From 527c36cbbb835f9c3a91e90edb105e57a2ac167f Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Mon, 22 Jun 2026 11:53:22 +0200 Subject: [PATCH 1/2] [uss_qualifier/resource] Add GeospatialModifier for FlightIntentsResource --- .basedpyright/baseline.json | 8 --- .../resources/flight_planning/__init__.py | 3 + .../flight_intents_resource.py | 67 +++++++++++++++++-- .../flight_intents_resource_test.py | 64 ++++++++++++++++++ 4 files changed, 130 insertions(+), 12 deletions(-) create mode 100644 monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource_test.py diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index fbebc2a460..eb665e5a18 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -5128,14 +5128,6 @@ "endColumn": 44, "lineCount": 1 } - }, - { - "code": "reportAttributeAccessIssue", - "range": { - "startColumn": 38, - "endColumn": 69, - "lineCount": 1 - } } ], "./monitoring/uss_qualifier/resources/flight_planning/flight_planners.py": [ diff --git a/monitoring/uss_qualifier/resources/flight_planning/__init__.py b/monitoring/uss_qualifier/resources/flight_planning/__init__.py index 35397e9f6c..d64273370e 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/__init__.py +++ b/monitoring/uss_qualifier/resources/flight_planning/__init__.py @@ -1,4 +1,7 @@ from .flight_intents_resource import FlightIntentsResource as FlightIntentsResource +from .flight_intents_resource import ( + FlightIntentsTriangularCascadeSoutheastResource as FlightIntentsTriangularCascadeSoutheastResource, +) from .flight_planners import ( FlightPlannerCombinationSelectorResource as FlightPlannerCombinationSelectorResource, ) diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource.py b/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource.py index e4189a1109..cf0aa3b138 100644 --- a/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource.py +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource.py @@ -1,22 +1,37 @@ +import json + +import s2sphere from implicitdict import ImplicitDict from monitoring.monitorlib.clients.flight_planning.flight_info_template import ( FlightInfoTemplate, ) +from monitoring.monitorlib.geo import ( + LatLngBoundingBox, + RelativeTranslation, + Transformation, +) +from monitoring.monitorlib.geotemporal import Volume4D from monitoring.uss_qualifier.resources.files import load_dict from monitoring.uss_qualifier.resources.flight_planning.flight_intent import ( FlightIntentCollection, FlightIntentID, FlightIntentsSpecification, ) +from monitoring.uss_qualifier.resources.geospatial import ( + GeospatialResource, + TriangularCascadeSoutheastResource, +) from monitoring.uss_qualifier.resources.resource import Resource -class FlightIntentsResource(Resource[FlightIntentsSpecification]): +class FlightIntentsResource(GeospatialResource, Resource[FlightIntentsSpecification]): + _spec: FlightIntentsSpecification _intent_collection: FlightIntentCollection def __init__(self, specification: FlightIntentsSpecification, resource_origin: str): super().__init__(specification, resource_origin) + self._spec = specification has_file = "file" in specification and specification.file has_literal = ( "intent_collection" in specification and specification.intent_collection @@ -34,17 +49,61 @@ def __init__(self, specification: FlightIntentsSpecification, resource_origin: s load_dict(specification.file), FlightIntentCollection ) elif has_literal: - self._intent_collection = specification.intent_collection + self._intent_collection = ImplicitDict.parse( + json.loads( + json.dumps(specification.intent_collection) + ), # NB: We need a copy to avoid sharing '_intent_collection' between instances + FlightIntentCollection, + ) if "transformations" in specification and specification.transformations: if ( "transformations" in self._intent_collection and self._intent_collection.transformations ): self._intent_collection.transformations.extend( - specification.transformations + specification.transformations[::] ) else: - self._intent_collection.transformations = specification.transformations + self._intent_collection.transformations = specification.transformations[ # NB: We do a copy to be independent between instances + :: + ] def get_flight_intents(self) -> dict[FlightIntentID, FlightInfoTemplate]: return self._intent_collection.resolve() + + def get_extents(self) -> LatLngBoundingBox: + rect = s2sphere.LatLngRect.empty() + for template in self.get_flight_intents().values(): + transformations = ( + template.transformations + if "transformations" in template and template.transformations + else [] + ) + for vt in template.basic_information.area: + v4d = Volume4D(volume=vt.resolve_3d()) + for transformation in transformations: + v4d = v4d.transform(transformation) + rect = rect.union(v4d.rect_bounds) + return LatLngBoundingBox.from_latlng_rect(rect) + + def move(self, meters_east: float, meters_north: float) -> "FlightIntentsResource": + new_spec = FlightIntentsSpecification(self._spec) + + transformation = Transformation( + relative_translation=RelativeTranslation( + meters_east=meters_east, + meters_north=meters_north, + ) + ) + + if "transformations" in new_spec and new_spec.transformations: + new_spec.transformations = new_spec.transformations + [transformation] + else: + new_spec.transformations = [transformation] + return FlightIntentsResource(new_spec, resource_origin=self.resource_origin) + + +class FlightIntentsTriangularCascadeSoutheastResource( + TriangularCascadeSoutheastResource[FlightIntentsResource] +): + pass diff --git a/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource_test.py b/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource_test.py new file mode 100644 index 0000000000..c4e9022d62 --- /dev/null +++ b/monitoring/uss_qualifier/resources/flight_planning/flight_intents_resource_test.py @@ -0,0 +1,64 @@ +import unittest + +from monitoring.monitorlib.geo import area_of_latlngrect +from monitoring.uss_qualifier.resources.definitions import ( + ResourceDeclaration, + ResourceID, +) +from monitoring.uss_qualifier.resources.geospatial import ( + TriangularCascadeSoutheastSpecification, +) +from monitoring.uss_qualifier.resources.resource import ( + create_resources, +) + + +class TestFlightIntentsTriangularCascadeSoutheastResource(unittest.TestCase): + def _build_declarations(self) -> dict[ResourceID, ResourceDeclaration]: + return { + "flight_intents": ResourceDeclaration( + resource_type="resources.flight_planning.FlightIntentsResource", + specification={ + "file": { + "path": "file://./test_data/che/flight_intents/general_flight_auth_flights.yaml", + }, + }, + ), + "flight_intents_modifier": ResourceDeclaration( + resource_type="resources.flight_planning.FlightIntentsTriangularCascadeSoutheastResource", + specification=TriangularCascadeSoutheastSpecification( + meters_east_margin=1000, meters_north_margin=1000 + ), + dependencies={ + "base_resource": "flight_intents", + }, + ), + } + + def test_overlap_only_for_same_index(self): + resources = create_resources(self._build_declarations(), "test", True) + modifier = resources["flight_intents_modifier"] + + extents = [ + modifier.provide_resource_for(index=i).get_extents() for i in range(11) + ] + base_area = area_of_latlngrect(extents[0].to_latlngrect()) + + for i in range(11): + for j in range(11): + overlap = area_of_latlngrect( + extents[i].to_latlngrect().intersection(extents[j].to_latlngrect()) + ) + if i == j: + assert ( + overlap > 0.99 * base_area + ), ( # Use 99% to compensate for errors + f"index {i}: self-overlap area {overlap:.2f}m² " + f"expected ~{base_area:.2f}m²" + ) + else: + assert ( + overlap < 0.01 * base_area + ), ( # Use 1% to compensate for errors + f"indices {i},{j}: unexpected overlap area {overlap:.2f}m²" + ) From a2046be568ab2034db4cd9ea7ab1000f30a522a7 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Wed, 20 May 2026 14:51:01 +0200 Subject: [PATCH 2/2] [uss_qualifier] Add ParallelFlightPlannerCombinations, parallel execution of actions generator and use everyting in f3548_21 --- .basedpyright/baseline.json | 8 -- .../uss_qualifier/action_generators/README.md | 23 ++++ .../flight_planning/README.md | 10 ++ .../flight_planning/__init__.py | 3 + .../flight_planning/planner_combinations.py | 54 ++++++-- .../dev/f3548_self_contained.yaml | 20 +++ .../configurations/dev/library/resources.yaml | 15 +++ .../configurations/dev/message_signing.yaml | 7 ++ .../configurations/dev/uspace.yaml | 11 ++ .../definitions/baseline_a.libsonnet | 25 ++++ .../uss_qualifier/suites/astm/utm/f3548_21.md | 6 +- .../suites/astm/utm/f3548_21.yaml | 16 ++- .../suites/faa/uft/message_signing.yaml | 8 ++ monitoring/uss_qualifier/suites/suite.py | 116 +++++++++++++++++- .../suites/uspace/flight_auth.yaml | 8 ++ .../suites/uspace/required_services.yaml | 8 ++ 16 files changed, 309 insertions(+), 29 deletions(-) diff --git a/.basedpyright/baseline.json b/.basedpyright/baseline.json index eb665e5a18..e5cb2096c0 100644 --- a/.basedpyright/baseline.json +++ b/.basedpyright/baseline.json @@ -4504,14 +4504,6 @@ "endColumn": 9, "lineCount": 3 } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 20, - "endColumn": 41, - "lineCount": 1 - } } ], "./monitoring/uss_qualifier/action_generators/interuss/mock_uss/with_locality.py": [ diff --git a/monitoring/uss_qualifier/action_generators/README.md b/monitoring/uss_qualifier/action_generators/README.md index ffe5fcda69..481b62f8de 100644 --- a/monitoring/uss_qualifier/action_generators/README.md +++ b/monitoring/uss_qualifier/action_generators/README.md @@ -3,3 +3,26 @@ The bulk of uss_qualifier's automated testing logic is contained in [test scenarios](../scenarios/README.md). A [test suite](../suites/README.md) is essentially a static "playlist" of test actions to perform (test scenarios, action generators, and other test suites), all of which ultimately resolve to test scenarios. An action generator is essentially a dynamic "playlist" of test actions -- it can generate test actions that vary according to provided resource values, situations, or other conditions only necessarily known at runtime. For documentation purposes, all action generators must statically declare the test actions they may take. However, whether each (or any) of these actions will actually be taken at runtime cannot be statically determined in general. + +## Parallel execution in action generators + +An action generator's `actions()` method yields one of: + +- a `TestSuiteAction` — executed sequentially, as before. +- a `list[list[TestSuiteAction]]` — a *parallel group*. The outer list holds the branches to execute concurrently; each inner list is a sequence of actions executed in order within its own branch. + +Example: + +```python +def actions(self) -> Iterator[TestSuiteAction | list[list[TestSuiteAction]]]: + yield base_action # sequential + yield [[A1, A2, A3], [B1, B2, B3]] # A and B in parallel +``` + +When a parallel group is yielded, each branch runs on its own thread. Reports are appended to the parent report in branch order. + +### Constraints + +Each branch shares the same `Resource` instances unless the action generator hands out distinct ones. If a resource has mutable state that two branches would race on, the generator must produce isolated copies - typically by declaring `ResourceModifier`-based variants and calling `.adjust(index)` for each branch. + +If a branch fails with `on_failure: Abort` (or hits a critical problem), the other branches are signalled to stop at the next action boundary. In-progress actions still finish. diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/README.md b/monitoring/uss_qualifier/action_generators/flight_planning/README.md index 75ecfce73a..a2c6da78be 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/README.md +++ b/monitoring/uss_qualifier/action_generators/flight_planning/README.md @@ -19,3 +19,13 @@ This action generator accepts a [FlightPlannersResource](../../resources/flight_ | `ussC` | `ussC` | `ExampleTestScenario` | The usage intent for this action generator is to enable design of simple test scenarios with a small number of participants, but to automatically repeat that simple scenario with all applicable role assignment combinations given a list of flight planner USSs to test. + +## `ParallelFlightPlannerCombinations` + +Variant of `FlightPlannerCombinations` that runs combinations in parallel where possible. + +Same configuration as `FlightPlannerCombinations`. The only difference is scheduling: combinations sharing no flight planner participant are grouped together and executed concurrently. Combinations sharing at least one participant remain in different groups (so no participant is hit by two tests at the same time). + +Groups are built greedily (first-fit): each combination is placed in the first existing group with no participant overlap, otherwise a new group is started. This is not minimal in the worst case but the problem is graph coloring, NP-hard. + +Each combination receives its own `adjust(index)` variant of every `ResourceModifier` resource in the pool (inherited from `FlightPlannerCombinations`), so parallel branches don't share mutable resource state. diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/__init__.py b/monitoring/uss_qualifier/action_generators/flight_planning/__init__.py index 97dfddf5af..8e8f798270 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/__init__.py +++ b/monitoring/uss_qualifier/action_generators/flight_planning/__init__.py @@ -1 +1,4 @@ from .planner_combinations import FlightPlannerCombinations as FlightPlannerCombinations +from .planner_combinations import ( + ParallelFlightPlannerCombinations as ParallelFlightPlannerCombinations, +) diff --git a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py index cf171c8872..21436e87e2 100644 --- a/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py +++ b/monitoring/uss_qualifier/action_generators/flight_planning/planner_combinations.py @@ -17,6 +17,7 @@ ) from monitoring.uss_qualifier.resources.resource import ( MissingResourceError, + ResourceProvidingResource, ResourceType, ) from monitoring.uss_qualifier.suites.definitions import TestSuiteActionDeclaration @@ -40,7 +41,7 @@ class FlightPlannerCombinationsSpecification(ImplicitDict): class FlightPlannerCombinations( ActionGenerator[FlightPlannerCombinationsSpecification] ): - _actions: list[TestSuiteAction] + _actions_with_participants: list[tuple[TestSuiteAction, frozenset[str]]] _current_action: int @classmethod @@ -91,8 +92,9 @@ def __init__( "default flight planner combination selector", ) - self._actions = [] + self._actions_with_participants = [] role_assignments = [0] * len(specification.roles) + combination_index = 0 while True: participants = flight_planners_resource.make_subset(role_assignments) flight_planners_combination = { @@ -100,13 +102,24 @@ def __init__( } if combination_selector.is_valid_combination(flight_planners_combination): - modified_resources = {k: v for k, v in resources.items()} + modified_resources = { + k: v.provide_resource_for(index=combination_index) + if isinstance(v, ResourceProvidingResource) + else v + for k, v in resources.items() + } for k, v in flight_planners_combination.items(): modified_resources[k] = v - self._actions.append( - TestSuiteAction(specification.action_to_repeat, modified_resources) + self._actions_with_participants.append( + ( + TestSuiteAction( + specification.action_to_repeat, modified_resources + ), + frozenset(p.participant_id for p in participants), + ) ) + combination_index += 1 index_to_increment = len(role_assignments) - 1 while index_to_increment >= 0: @@ -121,5 +134,32 @@ def __init__( self._current_action = 0 - def actions(self) -> Iterator[TestSuiteAction]: - yield from self._actions + def actions( + self, + ) -> Iterator[TestSuiteAction] | Iterator[list[list[TestSuiteAction]]]: + for action, _ in self._actions_with_participants: + yield action + + +class ParallelFlightPlannerCombinations(FlightPlannerCombinations): + """Like FlightPlannerCombinations, but yields actions grouped so actions + sharing no participant run in parallel.""" + + @classmethod + def get_name(cls) -> str: + return "For each appropriate combination of flight planner(s), in parallel where possible" + + def actions(self) -> Iterator[list[list[TestSuiteAction]]]: + # Greedy first-fit grouping + groups: list[list[tuple[TestSuiteAction, frozenset[str]]]] = [] + for action, participants in self._actions_with_participants: + for group in groups: + used = frozenset().union(*(p for _, p in group)) + if used.isdisjoint(participants): + group.append((action, participants)) + break + else: + groups.append([(action, participants)]) + + for group in groups: + yield [[action] for action, _ in group] diff --git a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml index e4fb34f94b..6b64148b3d 100644 --- a/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml +++ b/monitoring/uss_qualifier/configurations/dev/f3548_self_contained.yaml @@ -20,9 +20,13 @@ v1: flight_planners: flight_planners flight_planners_to_clear: flight_planners conflicting_flights: conflicting_flights + conflicting_flights_parallel: conflicting_flights_parallel priority_preemption_flights: conflicting_flights + priority_preemption_flights_parallel: conflicting_flights_parallel invalid_flight_intents: invalid_flight_intents + invalid_flight_intents_parallel: invalid_flight_intents_parallel non_conflicting_flights: non_conflicting_flights + non_conflicting_flights_parallel: non_conflicting_flights_parallel dss: dss dss_instances: dss_instances interuss_dss_instances: dss_instances @@ -253,6 +257,11 @@ v1: # Therefore, ground level is at roughly 93m above the WGS84 ellipsoid meters_up: 93 + conflicting_flights_parallel: + resource_type: resources.flight_planning.FlightIntentsModifier + dependencies: + base_resource: conflicting_flights + # Details of flights with invalid operational intents (used in flight intent validation scenario) invalid_flight_intents: resource_type: resources.flight_planning.FlightIntentsResource @@ -265,6 +274,12 @@ v1: degrees_east: -96.7587 meters_up: 93 + invalid_flight_intents_parallel: + resource_type: resources.flight_planning.FlightIntentsModifier + dependencies: + base_resource: invalid_flight_intents + + # Details of non-conflicting flights (used in data validation scenario) non_conflicting_flights: resource_type: resources.flight_planning.FlightIntentsResource @@ -278,6 +293,11 @@ v1: degrees_east: -96.7587 meters_up: 93 + non_conflicting_flights_parallel: + resource_type: resources.flight_planning.FlightIntentsModifier + dependencies: + base_resource: non_conflicting_flights + # How to execute a test run using this configuration execution: # Since we want to stop execution immediately if there are any unexpected failed checks, we set this parameter to diff --git a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml index 06e42eec7a..22e17aa6e4 100644 --- a/monitoring/uss_qualifier/configurations/dev/library/resources.yaml +++ b/monitoring/uss_qualifier/configurations/dev/library/resources.yaml @@ -347,6 +347,11 @@ che_conflicting_flights: degrees_east: 7.4774 meters_up: 605 +che_conflicting_flights_parallel: + resource_type: resources.flight_planning.FlightIntentsModifier + dependencies: + base_resource: che_conflicting_flights + che_invalid_flight_intents: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.flight_planning.FlightIntentsResource @@ -360,6 +365,11 @@ che_invalid_flight_intents: degrees_east: 7.4774 meters_up: 605 +che_invalid_flight_intents_parallel: + resource_type: resources.flight_planning.FlightIntentsModifier + dependencies: + base_resource: che_invalid_flight_intents + che_general_flight_auth_flights: $content_schema: monitoring/uss_qualifier/resources/definitions/ResourceDeclaration.json resource_type: resources.flight_planning.FlightIntentsResource @@ -381,6 +391,11 @@ che_non_conflicting_flights: degrees_east: 7.4774 meters_up: 605 +che_non_conflicting_flights_parallel: + resource_type: resources.flight_planning.FlightIntentsModifier + dependencies: + base_resource: che_non_conflicting_flights + # ===== General flight authorization ===== example_flight_check_table: diff --git a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml index 79e4d2aa4e..38b5aaf16c 100644 --- a/monitoring/uss_qualifier/configurations/dev/message_signing.yaml +++ b/monitoring/uss_qualifier/configurations/dev/message_signing.yaml @@ -3,8 +3,11 @@ v1: resources: resource_declarations: che_conflicting_flights: {$ref: 'library/resources.yaml#/che_conflicting_flights'} + che_conflicting_flights_parallel: {$ref: 'library/resources.yaml#/che_conflicting_flights_parallel'} che_invalid_flight_intents: {$ref: 'library/resources.yaml#/che_invalid_flight_intents'} + che_invalid_flight_intents_parallel: {$ref: 'library/resources.yaml#/che_invalid_flight_intents_parallel'} che_non_conflicting_flights: {$ref: 'library/resources.yaml#/che_non_conflicting_flights'} + che_non_conflicting_flights_parallel: {$ref: 'library/resources.yaml#/che_non_conflicting_flights_parallel'} che_problematically_big_area: {$ref: 'library/resources.yaml#/che_problematically_big_area'} che_planning_area_volume: {$ref: 'library/resources.yaml#/che_planning_area_volume'} che_planning_area: {$ref: 'library/resources.yaml#/che_planning_area'} @@ -55,9 +58,13 @@ v1: flight_planners: flight_planners combination_selector: combination_selector conflicting_flights: che_conflicting_flights + conflicting_flights_parallel: che_conflicting_flights_parallel invalid_flight_intents: che_invalid_flight_intents + invalid_flight_intents_parallel: che_invalid_flight_intents_parallel non_conflicting_flights: che_non_conflicting_flights + non_conflicting_flights_parallel: che_non_conflicting_flights_parallel priority_preemption_flights: che_conflicting_flights + priority_preemption_flights_parallel: che_conflicting_flights_parallel dss: scd_dss dss_instances: scd_dss_instances id_generator: id_generator diff --git a/monitoring/uss_qualifier/configurations/dev/uspace.yaml b/monitoring/uss_qualifier/configurations/dev/uspace.yaml index 7956294f78..86114cf25c 100644 --- a/monitoring/uss_qualifier/configurations/dev/uspace.yaml +++ b/monitoring/uss_qualifier/configurations/dev/uspace.yaml @@ -5,9 +5,12 @@ v1: resource_declarations: locality_che: {$ref: 'library/resources.yaml#/locality_che'} che_conflicting_flights: {$ref: 'library/resources.yaml#/che_conflicting_flights'} + che_conflicting_flights_parallel: {$ref: 'library/resources.yaml#/che_conflicting_flights_parallel'} che_invalid_flight_intents: {$ref: 'library/resources.yaml#/che_invalid_flight_intents'} + che_invalid_flight_intents_parallel: {$ref: 'library/resources.yaml#/che_invalid_flight_intents_parallel'} che_invalid_flight_auth_flights: {$ref: 'library/resources.yaml#/che_invalid_flight_auth_flights'} che_non_conflicting_flights: {$ref: 'library/resources.yaml#/che_non_conflicting_flights'} + che_non_conflicting_flights_parallel: {$ref: 'library/resources.yaml#/che_non_conflicting_flights_parallel'} che_planning_area_volume: {$ref: 'library/resources.yaml#/che_planning_area_volume'} che_planning_area: {$ref: 'library/resources.yaml#/che_planning_area'} netrid_observation_evaluation_configuration: {$ref: 'library/resources.yaml#/netrid_observation_evaluation_configuration'} @@ -59,10 +62,14 @@ v1: prod_env_version_providers: prod_env_version_providers? conflicting_flights: che_conflicting_flights + conflicting_flights_parallel: che_conflicting_flights_parallel priority_preemption_flights: che_conflicting_flights + priority_preemption_flights_parallel: che_conflicting_flights_parallel invalid_flight_intents: che_invalid_flight_intents + invalid_flight_intents_parallel: che_invalid_flight_intents_parallel invalid_flight_auth_flights: che_invalid_flight_auth_flights non_conflicting_flights: che_non_conflicting_flights + non_conflicting_flights_parallel: che_non_conflicting_flights_parallel flight_planners: all_flight_planners? mock_uss: mock_uss_instance_uss6 mock_uss_dp: mock_uss_instance_dp @@ -96,10 +103,14 @@ v1: prod_env_version_providers: prod_env_version_providers? conflicting_flights: conflicting_flights + conflicting_flights_parallel: conflicting_flights_parallel priority_preemption_flights: priority_preemption_flights + priority_preemption_flights_parallel: priority_preemption_flights_parallel invalid_flight_intents: invalid_flight_intents + invalid_flight_intents_parallel: invalid_flight_intents_parallel invalid_flight_auth_flights: invalid_flight_auth_flights non_conflicting_flights: non_conflicting_flights + non_conflicting_flights_parallel: non_conflicting_flights_parallel flight_planners: flight_planners? mock_uss: mock_uss mock_uss_dp: mock_uss_dp diff --git a/monitoring/uss_qualifier/configurations/dev/utm_implementation_us/definitions/baseline_a.libsonnet b/monitoring/uss_qualifier/configurations/dev/utm_implementation_us/definitions/baseline_a.libsonnet index 1bb0b0f8c5..1f9f531256 100644 --- a/monitoring/uss_qualifier/configurations/dev/utm_implementation_us/definitions/baseline_a.libsonnet +++ b/monitoring/uss_qualifier/configurations/dev/utm_implementation_us/definitions/baseline_a.libsonnet @@ -23,9 +23,13 @@ function(env) { flight_planners: 'flight_planners', flight_planners_to_clear: 'flight_planners_to_clear', conflicting_flights: 'conflicting_flights', + conflicting_flights_parallel: 'conflicting_flights_parallel', priority_preemption_flights: 'conflicting_flights', + priority_preemption_flights_parallel: 'conflicting_flights_parallel', invalid_flight_intents: 'invalid_flight_intents', + invalid_flight_intents_parallel: 'invalid_flight_intents_parallel', non_conflicting_flights: 'non_conflicting_flights', + non_conflicting_flights_parallel: 'non_conflicting_flights_parallel', test_exclusions: 'test_exclusions', dss: 'dss', dss_instances: 'dss_instances', @@ -205,6 +209,13 @@ function(env) { }, }, + conflicting_flights_parallel: { + resource_type: 'resources.flight_planning.FlightIntentsModifier', + dependencies: { + base_resource: 'conflicting_flights', + }, + }, + // Details of flights with invalid operational intents (used in flight intent validation scenario) invalid_flight_intents: { resource_type: 'resources.flight_planning.FlightIntentsResource', @@ -224,6 +235,13 @@ function(env) { }, }, + invalid_flight_intents_parallel: { + resource_type: 'resources.flight_planning.FlightIntentsModifier', + dependencies: { + base_resource: 'invalid_flight_intents', + }, + }, + // Details of non-conflicting flights (used in data validation scenario) non_conflicting_flights: { resource_type: 'resources.flight_planning.FlightIntentsResource', @@ -243,6 +261,13 @@ function(env) { }, }, + non_conflicting_flights_parallel: { + resource_type: 'resources.flight_planning.FlightIntentsModifier', + dependencies: { + base_resource: 'non_conflicting_flights', + }, + }, + // Name of the system under test for which the system version should be obtained from participants who provide version information system_identity: { resource_type: 'resources.versioning.SystemIdentityResource', diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md index 70649e6272..c5aac7544a 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.md @@ -9,11 +9,11 @@ 3. Scenario: [ASTM F3548 flight planners preparation](../../../scenarios/astm/utm/prep_planners.md) ([`scenarios.astm.utm.PrepareFlightPlanners`](../../../scenarios/astm/utm/prep_planners.py)) 4. Action generator: [`action_generators.astm.f3548.ForEachDSS`](../../../action_generators/astm/f3548/for_each_dss.py) 1. Suite: [DSS testing for ASTM F3548-21](dss_probing.md) ([`suites.astm.utm.dss_probing`](dss_probing.yaml)) -5. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) +5. Action generator: [`action_generators.flight_planning.ParallelFlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Validation of operational intents](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.md) ([`scenarios.astm.utm.FlightIntentValidation`](../../../scenarios/astm/utm/flight_intent_validation/flight_intent_validation.py)) -6. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) +6. Action generator: [`action_generators.flight_planning.ParallelFlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Nominal planning: conflict with higher priority](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.md) ([`scenarios.astm.utm.ConflictHigherPriority`](../../../scenarios/astm/utm/nominal_planning/conflict_higher_priority/conflict_higher_priority.py)) -7. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) +7. Action generator: [`action_generators.flight_planning.ParallelFlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Nominal planning: not permitted conflict with equal priority](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.md) ([`scenarios.astm.utm.ConflictEqualPriorityNotPermitted`](../../../scenarios/astm/utm/nominal_planning/conflict_equal_priority_not_permitted/conflict_equal_priority_not_permitted.py)) 8. Action generator: [`action_generators.flight_planning.FlightPlannerCombinations`](../../../action_generators/flight_planning/planner_combinations.py) 1. Scenario: [Data Validation of GET operational intents by USS](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.md) ([`scenarios.astm.utm.data_exchange_validation.GetOpResponseDataValidationByUSS`](../../../scenarios/astm/utm/data_exchange_validation/get_op_data_validation.py)) diff --git a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml index 04aea54e5d..b2bc2c0d87 100644 --- a/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml +++ b/monitoring/uss_qualifier/suites/astm/utm/f3548_21.yaml @@ -9,9 +9,13 @@ resources: interuss_dss_instances: resources.astm.f3548.v21.DSSInstancesResource? dss_datastore_cluster: resources.interuss.datastore.DatastoreDBClusterResource? conflicting_flights: resources.flight_planning.FlightIntentsResource + conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier priority_preemption_flights: resources.flight_planning.FlightIntentsResource? + priority_preemption_flights_parallel: resources.flight_planning.FlightIntentsModifier? invalid_flight_intents: resources.flight_planning.FlightIntentsResource + invalid_flight_intents_parallel: resources.flight_planning.FlightIntentsModifier non_conflicting_flights: resources.flight_planning.FlightIntentsResource + non_conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier nominal_planning_selector: resources.flight_planning.FlightPlannerCombinationSelectorResource? priority_planning_selector: resources.flight_planning.FlightPlannerCombinationSelectorResource? utm_auth: resources.communications.AuthAdapterResource? @@ -82,11 +86,11 @@ actions: dss_instance_id: dss on_failure: Continue - action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations + generator_type: action_generators.flight_planning.ParallelFlightPlannerCombinations resources: flight_planners: flight_planners flight_intent_validation_selector: flight_intent_validation_selector? - invalid_flight_intents: invalid_flight_intents + invalid_flight_intents: invalid_flight_intents_parallel dss: dss specification: action_to_repeat: @@ -103,11 +107,11 @@ actions: - uss1 on_failure: Continue - action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations + generator_type: action_generators.flight_planning.ParallelFlightPlannerCombinations resources: flight_planners: flight_planners priority_planning_selector: priority_planning_selector? - priority_preemption_flights: priority_preemption_flights + priority_preemption_flights: priority_preemption_flights_parallel? dss: dss specification: action_to_repeat: @@ -126,11 +130,11 @@ actions: - uss2 on_failure: Continue - action_generator: - generator_type: action_generators.flight_planning.FlightPlannerCombinations + generator_type: action_generators.flight_planning.ParallelFlightPlannerCombinations resources: flight_planners: flight_planners nominal_planning_selector: nominal_planning_selector? - conflicting_flights: conflicting_flights + conflicting_flights: conflicting_flights_parallel dss: dss specification: action_to_repeat: diff --git a/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml b/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml index c7f9259ae7..1680f0afc9 100644 --- a/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml +++ b/monitoring/uss_qualifier/suites/faa/uft/message_signing.yaml @@ -7,9 +7,13 @@ resources: dss_instances: resources.astm.f3548.v21.DSSInstancesResource? dss_datastore_cluster: resources.interuss.datastore.DatastoreDBClusterResource? conflicting_flights: resources.flight_planning.FlightIntentsResource + conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier non_conflicting_flights: resources.flight_planning.FlightIntentsResource + non_conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier priority_preemption_flights: resources.flight_planning.FlightIntentsResource + priority_preemption_flights_parallel: resources.flight_planning.FlightIntentsModifier invalid_flight_intents: resources.flight_planning.FlightIntentsResource + invalid_flight_intents_parallel: resources.flight_planning.FlightIntentsModifier id_generator: resources.interuss.IDGeneratorResource utm_client_identity: resources.communications.ClientIdentityResource second_utm_auth: resources.communications.AuthAdapterResource? @@ -27,12 +31,16 @@ actions: resources: mock_uss: mock_uss conflicting_flights: conflicting_flights + conflicting_flights_parallel: conflicting_flights_parallel non_conflicting_flights: non_conflicting_flights + non_conflicting_flights_parallel: non_conflicting_flights_parallel priority_preemption_flights: priority_preemption_flights + priority_preemption_flights_parallel: priority_preemption_flights_parallel flight_planners: flight_planners flight_planners_to_clear: flight_planners nominal_planning_selector: combination_selector invalid_flight_intents: invalid_flight_intents + invalid_flight_intents_parallel: invalid_flight_intents_parallel priority_planning_selector: combination_selector dss: dss dss_instances: dss_instances diff --git a/monitoring/uss_qualifier/suites/suite.py b/monitoring/uss_qualifier/suites/suite.py index bc921512fb..e251e7182b 100644 --- a/monitoring/uss_qualifier/suites/suite.py +++ b/monitoring/uss_qualifier/suites/suite.py @@ -3,7 +3,9 @@ import json import os import re +import threading from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import UTC, datetime @@ -155,7 +157,9 @@ def _run_test_scenario(self, context: ExecutionContext) -> TestScenarioReport: if not scenario: raise Exception("Cannot execute _run_test_scenario when no scenario is set") - logger.info(f'Running "{scenario.documentation.name}" scenario...') + logger.info( + f'Running "{scenario.documentation.name}" scenario{context.parallel_marker}...' + ) scenario.on_failed_check = _print_failed_check scenario.time_context[TimeDuringTest.StartOfTestRun] = Time(context.start_time) scenario.time_context[TimeDuringTest.StartOfScenario] = Time( @@ -185,7 +189,9 @@ def _run_test_scenario(self, context: ExecutionContext) -> TestScenarioReport: "\n".join(" " + line for line in lines), ) else: - logger.info(f'"{scenario.documentation.name}" scenario completed') + logger.info( + f'"{scenario.documentation.name}" scenario{context.parallel_marker} completed' + ) return report def _run_test_suite(self, context: ExecutionContext) -> TestSuiteReport: @@ -209,11 +215,95 @@ def _run_action_generator(self, context: ExecutionContext) -> ActionGeneratorRep start_time=StringBasedDateTime(arrow.utcnow()), ) - _run_actions(self.action_generator.actions(), context, report) + _run_generator_actions(self.action_generator.actions(), context, report) return report +def _run_generator_actions(actions, context, report): + success = True + for item in actions: + if isinstance(item, list): + sub_reports, branch_success = _run_parallel_branches(item, context) + report.actions.extend(sub_reports) + if not branch_success: + success = False + break + else: + action = item + if isinstance(action, SkippedActionReport): + action_report = TestSuiteActionReport(skipped_action=action) + elif context.should_stop_early_now(): + assert context.current_frame + action_report = TestSuiteActionReport( + skipped_action=SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow().datetime), + reason=TEST_RUN_TIMEOUT_SKIP_REASON, + declaration=context.current_frame.action.declaration, + ) + ) + else: + action_report = action.run(context) + report.actions.append(action_report) + if action_report.has_critical_problem(): + success = False + break + if not action_report.successful(): + success = False + if action.declaration.on_failure == ReactionToFailure.Abort: + break + report.successful = success + report.end_time = StringBasedDateTime(datetime.now(UTC)) + + +def _run_parallel_branches(branches, context): + parent_frame = context.current_frame + stop = threading.Event() + + def run_branch(branch): + context.current_frame = parent_frame + branch_reports = [] + branch_success = True + for action in branch: + if stop.is_set(): + break + if isinstance(action, SkippedActionReport): + ar = TestSuiteActionReport(skipped_action=action) + elif context.should_stop_early_now(): + assert context.current_frame + ar = TestSuiteActionReport( + skipped_action=SkippedActionReport( + timestamp=StringBasedDateTime(arrow.utcnow().datetime), + reason=TEST_RUN_TIMEOUT_SKIP_REASON, + declaration=context.current_frame.action.declaration, + ) + ) + else: + ar = action.run(context) + branch_reports.append(ar) + if ar.has_critical_problem(): + branch_success = False + stop.set() + break + if not ar.successful(): + branch_success = False + if action.declaration.on_failure == ReactionToFailure.Abort: + stop.set() + break + return branch_reports, branch_success + + def run_branch_indexed(i, branch): + context.parallel_marker = f" [{i + 1}/{len(branches)}]" + return run_branch(branch) + + with ThreadPoolExecutor(max_workers=min(len(branches), 8)) as ex: + results = list(ex.map(run_branch_indexed, range(len(branches)), branches)) + + flat_reports = [r for branch_reports, _ in results for r in branch_reports] + overall_success = all(s for _, s in results) + return flat_reports, overall_success + + class TestSuite: declaration: TestSuiteDeclaration definition: TestSuiteDefinition @@ -425,7 +515,7 @@ class ExecutionContext: config: ExecutionConfiguration | None acceptable_findings: list[FullyQualifiedCheck] top_frame: ActionStackFrame | None - current_frame: ActionStackFrame | None + _current_frame: threading.local def __init__( self, @@ -435,8 +525,24 @@ def __init__( self.config = config self.acceptable_findings = acceptable_findings self.top_frame = None - self.current_frame = None self.start_time = arrow.utcnow().datetime + self._current_frame = threading.local() + + @property + def current_frame(self) -> ActionStackFrame | None: + return getattr(self._current_frame, "value", None) + + @current_frame.setter + def current_frame(self, value: ActionStackFrame | None) -> None: + self._current_frame.value = value + + @property + def parallel_marker(self) -> str: + return getattr(self._current_frame, "parallel_marker", "") + + @parallel_marker.setter + def parallel_marker(self, value: str) -> None: + self._current_frame.parallel_marker = value def sibling_queries(self) -> Iterator[Query]: if self.current_frame is None or self.current_frame.parent is None: diff --git a/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml b/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml index 6da7561b86..53c0016e31 100644 --- a/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml +++ b/monitoring/uss_qualifier/suites/uspace/flight_auth.yaml @@ -3,10 +3,14 @@ resources: test_env_version_providers: resources.versioning.VersionProvidersResource? prod_env_version_providers: resources.versioning.VersionProvidersResource? conflicting_flights: resources.flight_planning.FlightIntentsResource + conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier priority_preemption_flights: resources.flight_planning.FlightIntentsResource + priority_preemption_flights_parallel: resources.flight_planning.FlightIntentsModifier invalid_flight_auth_flights: resources.flight_planning.FlightIntentsResource invalid_flight_intents: resources.flight_planning.FlightIntentsResource + invalid_flight_intents_parallel: resources.flight_planning.FlightIntentsModifier non_conflicting_flights: resources.flight_planning.FlightIntentsResource + non_conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier flight_planners: resources.flight_planning.FlightPlannersResource? mock_uss: resources.interuss.mock_uss.client.MockUSSResource? dss: resources.astm.f3548.v21.DSSInstanceResource @@ -27,9 +31,13 @@ actions: test_env_version_providers: test_env_version_providers? prod_env_version_providers: prod_env_version_providers? conflicting_flights: conflicting_flights + conflicting_flights_parallel: conflicting_flights_parallel priority_preemption_flights: priority_preemption_flights + priority_preemption_flights_parallel: priority_preemption_flights_parallel invalid_flight_intents: invalid_flight_intents + invalid_flight_intents_parallel: invalid_flight_intents_parallel non_conflicting_flights: non_conflicting_flights + non_conflicting_flights_parallel: non_conflicting_flights_parallel flight_planners: flight_planners? flight_planners_to_clear: flight_planners? mock_uss: mock_uss diff --git a/monitoring/uss_qualifier/suites/uspace/required_services.yaml b/monitoring/uss_qualifier/suites/uspace/required_services.yaml index 7a020b5a78..e09023c689 100644 --- a/monitoring/uss_qualifier/suites/uspace/required_services.yaml +++ b/monitoring/uss_qualifier/suites/uspace/required_services.yaml @@ -4,10 +4,14 @@ resources: prod_env_version_providers: resources.versioning.VersionProvidersResource? conflicting_flights: resources.flight_planning.FlightIntentsResource + conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier priority_preemption_flights: resources.flight_planning.FlightIntentsResource + priority_preemption_flights_parallel: resources.flight_planning.FlightIntentsModifier invalid_flight_intents: resources.flight_planning.FlightIntentsResource + invalid_flight_intents_parallel: resources.flight_planning.FlightIntentsModifier invalid_flight_auth_flights: resources.flight_planning.FlightIntentsResource non_conflicting_flights: resources.flight_planning.FlightIntentsResource + non_conflicting_flights_parallel: resources.flight_planning.FlightIntentsModifier flight_planners: resources.flight_planning.FlightPlannersResource? mock_uss: resources.interuss.mock_uss.client.MockUSSResource? mock_uss_dp: resources.interuss.mock_uss.client.MockUSSResource? @@ -49,10 +53,14 @@ actions: test_env_version_providers: test_env_version_providers? prod_env_version_providers: prod_env_version_providers? conflicting_flights: conflicting_flights + conflicting_flights_parallel: conflicting_flights_parallel priority_preemption_flights: priority_preemption_flights + priority_preemption_flights_parallel: priority_preemption_flights_parallel invalid_flight_intents: invalid_flight_intents + invalid_flight_intents_parallel: invalid_flight_intents_parallel invalid_flight_auth_flights: invalid_flight_auth_flights non_conflicting_flights: non_conflicting_flights + non_conflicting_flights_parallel: non_conflicting_flights_parallel flight_planners: flight_planners? mock_uss: mock_uss dss: scd_dss