feat(sort): multi-reference minimizer scoring for dataset suggestion

Cargo.toml

@@ -24,6 +24,7 @@ authors = ["Nextstrain team"]
 
 
 [workspace.dependencies]
+approx = "=0.5.1"
 assert2 = "=0.3.16"
 auto_ops = "=0.3.0"
 base64 = "=0.22.1"

packages/nextclade-schemas/input-pathogen-json.schema.json

@@ -185,6 +185,17 @@
         }
       ]
     },
+    "minimizerIndex": {
+      "description": "Reference sequences used to build this dataset's entry in the shared minimizer index for dataset suggestion. Build-time metadata: read when generating the index, ignored during analysis.",
+      "anyOf": [
+        {
+          "$ref": "#/definitions/MinimizerIndexConfig"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
     "defaultCds": {
       "description": "CDS shown by default in the Nextclade Web sequence view (e.g. \"S\", \"HA1\").",
       "type": [
@@ -443,6 +454,19 @@
         }
       }
     },
+    "MinimizerIndexConfig": {
+      "description": "Configuration for building this dataset's entry in the shared minimizer index used for dataset suggestion (`nextclade sort`).",
+      "type": "object",
+      "properties": {
+        "references": {
+          "description": "FASTA files, relative to the dataset directory, whose sequences all contribute minimizers to this dataset's suggestion fingerprint. When absent, the dataset's main reference sequence is used.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "LabelledMutationsConfig": {
       "description": "Mapping from specific mutations to human-readable labels (e.g. clade-defining or drug-resistance mutations).",
       "examples": [

packages/nextclade-schemas/input-pathogen-json.schema.yaml

@@ -119,6 +119,11 @@ properties:
     description: Filenames of other dataset input files (reference FASTA, genome annotation, tree, examples).
     allOf:
     - $ref: '#/definitions/DatasetFiles'
+  minimizerIndex:
+    description: 'Reference sequences used to build this dataset''s entry in the shared minimizer index for dataset suggestion. Build-time metadata: read when generating the index, ignored during analysis.'
+    anyOf:
+    - $ref: '#/definitions/MinimizerIndexConfig'
+    - type: 'null'
   defaultCds:
     description: CDS shown by default in the Nextclade Web sequence view (e.g. "S", "HA1").
     type:
@@ -278,6 +283,15 @@ definitions:
         type:
         - string
         - 'null'
+  MinimizerIndexConfig:
+    description: Configuration for building this dataset's entry in the shared minimizer index used for dataset suggestion (`nextclade sort`).
+    type: object
+    properties:
+      references:
+        description: FASTA files, relative to the dataset directory, whose sequences all contribute minimizers to this dataset's suggestion fingerprint. When absent, the dataset's main reference sequence is used.
+        type: array
+        items:
+          type: string
   LabelledMutationsConfig:
     description: Mapping from specific mutations to human-readable labels (e.g. clade-defining or drug-resistance mutations).
     examples:

packages/nextclade-schemas/internal-minimizer-index-json.schema.json

@@ -77,8 +77,18 @@
           "type": "string"
         },
         "nMinimizers": {
+          "description": "Deprecated: prefer `expected_minimizer_hits`. Retained as the integer score denominator for older clients. For indexes that provide both fields it is the rounded `expected_minimizer_hits`; older indexes instead carry the literal count of stored minimizers.",
+          "deprecated": true,
           "type": "integer",
           "format": "int64"
+        },
+        "expectedMinimizerHits": {
+          "description": "Exact fractional expected number of minimizer hits from a single reference. Preferred as the score denominator when present; absent in older indexes, which fall back to `n_minimizers`.",
+          "type": [
+            "number",
+            "null"
+          ],
+          "format": "double"
         }
       }
     }

packages/nextclade-schemas/internal-minimizer-index-json.schema.yaml

@@ -56,5 +56,13 @@ definitions:
       name:
         type: string
       nMinimizers:
+        description: 'Deprecated: prefer `expected_minimizer_hits`. Retained as the integer score denominator for older clients. For indexes that provide both fields it is the rounded `expected_minimizer_hits`; older indexes instead carry the literal count of stored minimizers.'
+        deprecated: true
         type: integer
         format: int64
+      expectedMinimizerHits:
+        description: Exact fractional expected number of minimizer hits from a single reference. Preferred as the score denominator when present; absent in older indexes, which fall back to `n_minimizers`.
+        type:
+        - number
+        - 'null'
+        format: double

packages/nextclade-schemas/nextclade-auspice-extensions.schema.json

@@ -519,6 +519,17 @@
             }
           ]
         },
+        "minimizerIndex": {
+          "description": "Reference sequences used to build this dataset's entry in the shared minimizer index for dataset suggestion. Build-time metadata: read when generating the index, ignored during analysis.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/MinimizerIndexConfig"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "defaultCds": {
           "description": "CDS shown by default in the Nextclade Web sequence view (e.g. \"S\", \"HA1\").",
           "type": [
@@ -777,6 +788,19 @@
         }
       }
     },
+    "MinimizerIndexConfig": {
+      "description": "Configuration for building this dataset's entry in the shared minimizer index used for dataset suggestion (`nextclade sort`).",
+      "type": "object",
+      "properties": {
+        "references": {
+          "description": "FASTA files, relative to the dataset directory, whose sequences all contribute minimizers to this dataset's suggestion fingerprint. When absent, the dataset's main reference sequence is used.",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "LabelledMutationsConfig": {
       "description": "Mapping from specific mutations to human-readable labels (e.g. clade-defining or drug-resistance mutations).",
       "examples": [

packages/nextclade-schemas/nextclade-auspice-extensions.schema.yaml

@@ -339,6 +339,11 @@ definitions:
         description: Filenames of other dataset input files (reference FASTA, genome annotation, tree, examples).
         allOf:
         - $ref: '#/definitions/DatasetFiles'
+      minimizerIndex:
+        description: 'Reference sequences used to build this dataset''s entry in the shared minimizer index for dataset suggestion. Build-time metadata: read when generating the index, ignored during analysis.'
+        anyOf:
+        - $ref: '#/definitions/MinimizerIndexConfig'
+        - type: 'null'
       defaultCds:
         description: CDS shown by default in the Nextclade Web sequence view (e.g. "S", "HA1").
         type:
@@ -497,6 +502,15 @@ definitions:
         type:
         - string
         - 'null'
+  MinimizerIndexConfig:
+    description: Configuration for building this dataset's entry in the shared minimizer index used for dataset suggestion (`nextclade sort`).
+    type: object
+    properties:
+      references:
+        description: FASTA files, relative to the dataset directory, whose sequences all contribute minimizers to this dataset's suggestion fingerprint. When absent, the dataset's main reference sequence is used.
+        type: array
+        items:
+          type: string
   LabelledMutationsConfig:
     description: Mapping from specific mutations to human-readable labels (e.g. clade-defining or drug-resistance mutations).
     examples:

packages/nextclade/Cargo.toml

@@ -85,6 +85,7 @@ zip = { workspace = true }
 zstd = { workspace = true }
 
 [dev-dependencies]
+approx = { workspace = true }
 assert2 = { workspace = true }
 criterion = { workspace = true }
 rstest = { workspace = true }

packages/nextclade/src/analyze/virus_properties.rs

@@ -79,6 +79,10 @@ pub struct VirusProperties {
   #[serde(default, skip_serializing_if = "DatasetFiles::is_default")]
   pub files: DatasetFiles,
 
+  /// Reference sequences used to build this dataset's entry in the shared minimizer index for dataset suggestion. Build-time metadata: read when generating the index, ignored during analysis.
+  #[serde(default, skip_serializing_if = "Option::is_none")]
+  pub minimizer_index: Option<MinimizerIndexConfig>,
+
   /// CDS shown by default in the Nextclade Web sequence view (e.g. "S", "HA1").
   #[serde(default, skip_serializing_if = "Option::is_none")]
   pub default_cds: Option<String>,
@@ -138,6 +142,18 @@ pub struct VirusProperties {
   pub other: serde_json::Value,
 }
 
+/// Configuration for building this dataset's entry in the shared minimizer index used for dataset suggestion (`nextclade sort`).
+#[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct MinimizerIndexConfig {
+  /// FASTA files, relative to the dataset directory, whose sequences all contribute minimizers to this dataset's suggestion fingerprint. When absent, the dataset's main reference sequence is used.
+  #[serde(default, skip_serializing_if = "Vec::is_empty")]
+  pub references: Vec<String>,
+
+  #[serde(flatten)]
+  pub other: serde_json::Value,
+}
+
 /// Contact and documentation URLs for dataset maintenance
 #[derive(Clone, Default, Debug, Eq, PartialEq, Serialize, Deserialize, schemars::JsonSchema)]
 #[serde(rename_all = "camelCase")]
@@ -412,6 +428,7 @@ impl VirusProperties {
       shortcuts: vec_of_owned!["flu_h3n2_ha_broad", "nextstrain/flu/h3n2/ha/wisconsin-67-2005"],
       meta: DatasetMeta::default(),
       files: DatasetFiles::default(),
+      minimizer_index: None,
       default_cds: Some(o!("HA1")),
       cds_order_preference: vec_of_owned!["HA1", "HA2"],
       mut_labels: LabelledMutationsConfig::example(),
@@ -457,3 +474,27 @@ impl VirusProperties {
       .is_none_or(|compat| compat.is_cli_compatible(current_cli_version))
   }
 }
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+  use pretty_assertions::assert_eq;
+
+  #[test]
+  fn test_virus_properties_parses_minimizer_index() {
+    let json = r#"{"schemaVersion":"3.0.0","minimizerIndex":{"references":["minimizer_refs/additional_refs.fasta"]}}"#;
+    let props = VirusProperties::from_str(&json).unwrap();
+    let mi = props
+      .minimizer_index
+      .expect("minimizerIndex should parse into the typed field");
+    assert_eq!(vec_of_owned!["minimizer_refs/additional_refs.fasta"], mi.references);
+    assert!(props.other.get("minimizerIndex").is_none());
+  }
+
+  #[test]
+  fn test_virus_properties_minimizer_index_absent_is_none() {
+    let json = r#"{"schemaVersion":"3.0.0"}"#;
+    let props = VirusProperties::from_str(&json).unwrap();
+    assert_eq!(None, props.minimizer_index);
+  }
+}

packages/nextclade/src/sort/minimizer_index.rs

@@ -1,6 +1,7 @@
 use crate::io::fs::read_file_to_string;
 use crate::io::json::json_parse;
 use crate::io::schema_version::{SchemaVersion, SchemaVersionParams};
+use crate::make_error;
 use eyre::{Report, WrapErr};
 use log::warn;
 use schemars::JsonSchema;
@@ -84,8 +85,18 @@ pub struct MinimizerIndexParams {
 pub struct MinimizerIndexRefInfo {
   pub length: i64,
   pub name: String,
+
+  /// Deprecated: prefer `expected_minimizer_hits`. Retained as the integer score denominator for older
+  /// clients. For indexes that provide both fields it is the rounded `expected_minimizer_hits`; older
+  /// indexes instead carry the literal count of stored minimizers.
+  #[deprecated(note = "use `expected_minimizer_hits`; retained for backward compatibility with older indexes")]
   pub n_minimizers: i64,
 
+  /// Exact fractional expected number of minimizer hits from a single reference. Preferred as the score
+  /// denominator when present; absent in older indexes, which fall back to `n_minimizers`.
+  #[serde(default, skip_serializing_if = "Option::is_none")]
+  pub expected_minimizer_hits: Option<f64>,
+
   #[serde(flatten)]
   pub other: serde_json::Value,
 }
@@ -135,15 +146,111 @@ impl MinimizerIndexJson {
       );
     }
 
-    json_parse(s).wrap_err("When parsing minimizer index")
+    let index: Self = json_parse(s).wrap_err("When parsing minimizer index")?;
+    index.validate()?;
+    Ok(index)
+  }
+
+  /// Reject a malformed minimizer index before it reaches scoring. The index is external data (fetched
+  /// from a dataset server or supplied via `--input-minimizer-index-json`), so its numeric fields carry
+  /// no type-level guarantees: a negative `k`/`cutoff` casts to a huge unsigned value, and a
+  /// non-positive or non-finite denominator produces `inf`/`NaN` scores that silently win or vanish from
+  /// dataset selection. Fail loudly instead.
+  fn validate(&self) -> Result<(), Report> {
+    if self.params.k <= 0 {
+      return make_error!(
+        "Minimizer index parameter `k` must be positive, but found {}",
+        self.params.k
+      );
+    }
+    if !(0 < self.params.cutoff && self.params.cutoff <= (1_i64 << 32)) {
+      return make_error!(
+        "Minimizer index parameter `cutoff` must be in (0, 2^32], but found {}",
+        self.params.cutoff
+      );
+    }
+    for reference in &self.references {
+      if reference.length <= 0 {
+        return make_error!(
+          "Minimizer index reference '{}' has non-positive length {}",
+          reference.name,
+          reference.length
+        );
+      }
+      #[allow(deprecated)] // reads the fallback denominator to validate it, same precedence as scoring
+      let denominator = reference
+        .expected_minimizer_hits
+        .unwrap_or(reference.n_minimizers as f64);
+      if !denominator.is_finite() || denominator <= 0.0 {
+        return make_error!(
+          "Minimizer index reference '{}' has a non-positive or non-finite score denominator ({denominator})",
+          reference.name
+        );
+      }
+    }
+    Ok(())
   }
 }
 
 #[cfg(test)]
 mod tests {
   use super::*;
+  use pretty_assertions::assert_eq;
   use rstest::rstest;
 
+  // Minimal minimizer index JSON with one reference. `expected` is spliced verbatim (e.g.
+  // `,"expectedMinimizerHits":436.4375` or empty) so both presence and absence can be tested.
+  fn index_json(k: i64, cutoff: i64, length: i64, n_minimizers: i64, expected: &str) -> String {
+    format!(
+      r#"{{"schemaVersion":"3.0.0","version":"1","params":{{"k":{k},"cutoff":{cutoff}}},"references":[{{"length":{length},"name":"ds","nMinimizers":{n_minimizers}{expected}}}]}}"#
+    )
+  }
+
+  #[test]
+  fn test_minimizer_index_deserializes_expected_minimizer_hits() {
+    let idx = MinimizerIndexJson::from_str(index_json(
+      17,
+      1 << 28,
+      7000,
+      436,
+      r#","expectedMinimizerHits":436.4375"#,
+    ))
+    .unwrap();
+    assert_eq!(Some(436.4375), idx.references[0].expected_minimizer_hits);
+  }
+
+  #[test]
+  fn test_minimizer_index_absent_expected_minimizer_hits_deserializes_to_none() {
+    let idx = MinimizerIndexJson::from_str(index_json(17, 1 << 28, 7000, 436, "")).unwrap();
+    assert_eq!(None, idx.references[0].expected_minimizer_hits);
+    // the camelCase key did not silently land in the flatten catch-all
+    assert!(idx.references[0].other.get("expectedMinimizerHits").is_none());
+  }
+
+  #[test]
+  fn test_minimizer_index_roundtrip_omits_none_expected_minimizer_hits() {
+    let idx = MinimizerIndexJson::from_str(index_json(17, 1 << 28, 7000, 436, "")).unwrap();
+    let serialized = serde_json::to_string(&idx).unwrap();
+    assert!(!serialized.contains("expectedMinimizerHits"));
+  }
+
+  #[rustfmt::skip]
+  #[rstest]
+  #[case::valid(                 index_json(17, 1 << 28,  7000, 436, r#","expectedMinimizerHits":436.4375"#), true)]
+  #[case::valid_int_fallback(    index_json(17, 1 << 28,  7000, 436, ""),                                     true)]
+  #[case::negative_k(            index_json(-1, 1 << 28,  7000, 436, ""),                                     false)]
+  #[case::zero_k(                index_json(0,  1 << 28,  7000, 436, ""),                                     false)]
+  #[case::negative_cutoff(       index_json(17, -1,       7000, 436, ""),                                     false)]
+  #[case::zero_cutoff(           index_json(17, 0,        7000, 436, ""),                                     false)]
+  #[case::zero_length(           index_json(17, 1 << 28,  0,    436, ""),                                     false)]
+  #[case::zero_denominator(      index_json(17, 1 << 28,  7000, 0,   ""),                                     false)]
+  #[case::negative_expected(     index_json(17, 1 << 28,  7000, 436, r#","expectedMinimizerHits":-1.0"#),     false)]
+  #[case::zero_expected(         index_json(17, 1 << 28,  7000, 436, r#","expectedMinimizerHits":0.0"#),      false)]
+  #[trace]
+  fn test_minimizer_index_validate(#[case] json: String, #[case] ok: bool) {
+    assert_eq!(ok, MinimizerIndexJson::from_str(&json).is_ok(), "for: {json}");
+  }
+
   #[rustfmt::skip]
   #[rstest]
   #[case::equal(           "1",  false)]