Skip to content

Commit bc69490

Browse files
committed
Support data disk for shared image gallery transformer
1 parent dca0133 commit bc69490

4 files changed

Lines changed: 160 additions & 24 deletions

File tree

lisa/sut_orchestrator/azure/arm_template.bicep

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -473,16 +473,18 @@ resource nodes_vms 'Microsoft.Compute/virtualMachines@2024-03-01' = [for i in ra
473473
imageReference: getImageReference(nodes[i])
474474
osDisk: getVMOsDisk(nodes[i])
475475
diskControllerType: (nodes[i].disk_controller_type == 'SCSI') ? null : nodes[i].disk_controller_type
476-
dataDisks: concat(
477-
map(
478-
filter(range(0, length(data_disks)), j => !shouldAttachDataDisk(data_disks[j])),
479-
j => getCreateDisk(data_disks[j], '${nodes[i].name}-data-disk-${j}', j)
480-
),
481-
map(
482-
filter(range(0, length(data_disks)), j => shouldAttachDataDisk(data_disks[j])),
483-
j => getAttachDisk(data_disks[j], '${nodes[i].name}-data-disk-${j}', j)
476+
dataDisks: empty(data_disks)
477+
? null
478+
: concat(
479+
map(
480+
filter(range(0, length(data_disks)), j => !shouldAttachDataDisk(data_disks[j])),
481+
j => getCreateDisk(data_disks[j], '${nodes[i].name}-data-disk-${j}', j)
482+
),
483+
map(
484+
filter(range(0, length(data_disks)), j => shouldAttachDataDisk(data_disks[j])),
485+
j => getAttachDisk(data_disks[j], '${nodes[i].name}-data-disk-${j}', j)
486+
)
484487
)
485-
)
486488
}
487489
networkProfile: {
488490
networkInterfaces: [for j in range(0, nodes[i].nic_count): {

lisa/sut_orchestrator/azure/autogen_arm_template.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -816,7 +816,7 @@
816816
"imageReference": "[__bicep.getImageReference(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]])]",
817817
"osDisk": "[__bicep.getVMOsDisk(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]])]",
818818
"diskControllerType": "[if(equals(parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].disk_controller_type, 'SCSI'), null(), parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].disk_controller_type)]",
819-
"dataDisks": "[concat(map(filter(range(0, length(parameters('data_disks'))), lambda('j', not(__bicep.shouldAttachDataDisk(parameters('data_disks')[lambdaVariables('j')])))), lambda('j', __bicep.getCreateDisk(parameters('data_disks')[lambdaVariables('j')], format('{0}-data-disk-{1}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name, lambdaVariables('j')), lambdaVariables('j')))), map(filter(range(0, length(parameters('data_disks'))), lambda('j', __bicep.shouldAttachDataDisk(parameters('data_disks')[lambdaVariables('j')]))), lambda('j', __bicep.getAttachDisk(parameters('data_disks')[lambdaVariables('j')], format('{0}-data-disk-{1}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name, lambdaVariables('j')), lambdaVariables('j')))))]"
819+
"dataDisks": "[if(empty(parameters('data_disks')), null(), concat(map(filter(range(0, length(parameters('data_disks'))), lambda('j', not(__bicep.shouldAttachDataDisk(parameters('data_disks')[lambdaVariables('j')])))), lambda('j', __bicep.getCreateDisk(parameters('data_disks')[lambdaVariables('j')], format('{0}-data-disk-{1}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name, lambdaVariables('j')), lambdaVariables('j')))), map(filter(range(0, length(parameters('data_disks'))), lambda('j', __bicep.shouldAttachDataDisk(parameters('data_disks')[lambdaVariables('j')]))), lambda('j', __bicep.getAttachDisk(parameters('data_disks')[lambdaVariables('j')], format('{0}-data-disk-{1}', parameters('nodes')[range(0, variables('node_count'))[copyIndex()]].name, lambdaVariables('j')), lambdaVariables('j'))))))]"
820820
},
821821
"networkProfile": {
822822
"copy": [

lisa/sut_orchestrator/azure/common.py

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2971,6 +2971,7 @@ def check_or_create_gallery_image_version(
29712971
vhd_resource_group_name: str,
29722972
vhd_storage_account_name: str,
29732973
gallery_image_target_regions: List[str],
2974+
data_vhd_paths: Optional[List[Dict[str, Any]]] = None,
29742975
) -> None:
29752976
try:
29762977
compute_client = get_compute_client(platform)
@@ -2992,7 +2993,7 @@ def check_or_create_gallery_image_version(
29922993
"storage_account_type": storage_account_type,
29932994
}
29942995
)
2995-
image_version_post_body = {
2996+
image_version_post_body: Dict[str, Any] = {
29962997
"location": gallery_image_location,
29972998
"publishing_profile": {"target_regions": target_regions},
29982999
"storageProfile": {
@@ -3010,6 +3011,50 @@ def check_or_create_gallery_image_version(
30103011
},
30113012
},
30123013
}
3014+
3015+
if data_vhd_paths:
3016+
data_disk_images: List[Dict[str, Any]] = []
3017+
assigned_luns: set[int] = set()
3018+
for index, data_vhd_path_item in enumerate(data_vhd_paths):
3019+
lun = index
3020+
raw_path = data_vhd_path_item.get("url")
3021+
if not isinstance(raw_path, str) or not raw_path:
3022+
continue
3023+
data_vhd_path = raw_path
3024+
3025+
raw_lun = data_vhd_path_item.get("lun")
3026+
if isinstance(raw_lun, int) and raw_lun >= 0:
3027+
lun = raw_lun
3028+
elif isinstance(raw_lun, str) and raw_lun.strip().isdigit():
3029+
lun = int(raw_lun.strip())
3030+
3031+
if lun in assigned_luns:
3032+
raise LisaException(
3033+
f"duplicated data disk lun '{lun}' in data_vhd_paths"
3034+
)
3035+
assigned_luns.add(lun)
3036+
3037+
data_vhd_details = get_vhd_details(platform, data_vhd_path)
3038+
data_disk_images.append(
3039+
{
3040+
"lun": lun,
3041+
"hostCaching": host_caching_type,
3042+
"source": {
3043+
"uri": data_vhd_path,
3044+
"storageAccountId": (
3045+
f"/subscriptions/{platform.subscription_id}/"
3046+
f"resourceGroups/"
3047+
f"{data_vhd_details['resource_group_name']}"
3048+
"/providers/Microsoft.Storage/storageAccounts/"
3049+
f"{data_vhd_details['account_name']}"
3050+
),
3051+
},
3052+
}
3053+
)
3054+
image_version_post_body["storageProfile"][
3055+
"dataDiskImages"
3056+
] = data_disk_images
3057+
30133058
operation = compute_client.gallery_image_versions.begin_create_or_update(
30143059
gallery_resource_group_name,
30153060
gallery_name,

lisa/sut_orchestrator/azure/transformers.py

Lines changed: 102 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
get_date_str,
2525
get_datetime_path,
2626
)
27+
from lisa.secret import PATTERN_URL, add_secret
2728

2829
from .common import (
2930
AZURE_SHARED_RG_NAME,
@@ -481,9 +482,21 @@ class SigTransformerSchema(schema.Transformer):
481482
azure_sig_url: shared_gallery
482483
"""
483484

484-
# raw vhd URL, it can be the blob under the same subscription of SIG
485-
# or SASURL
486-
vhd: str = field(default="", metadata=field_metadata(required=True))
485+
# Raw VHD URL or VHD schema object. String form keeps backward compatibility:
486+
# vhd: "https://.../os.vhd"
487+
# Object form supports data disk VHDs:
488+
# vhd:
489+
# vhd_path: "https://.../os.vhd"
490+
# data_vhd_paths:
491+
# - data_vhd:
492+
# lun: 0
493+
# url: "https://.../data0.vhd"
494+
# - data_vhd:
495+
# lun: 1
496+
# url: "https://.../data1.vhd"
497+
vhd: Union[str, Dict[Any, Any]] = field(
498+
default="", metadata=field_metadata(required=True)
499+
)
487500
# if not specify gallery_resource_group_name, use shared resource group name
488501
gallery_resource_group_name: str = field(default=AZURE_SHARED_RG_NAME)
489502
# if not specified, will use the first location of gallery image
@@ -610,18 +623,24 @@ def _internal_run(self) -> Dict[str, Any]:
610623
runbook.gallery_resource_group_location = image_location
611624
if not runbook.gallery_location:
612625
runbook.gallery_location = image_location
626+
627+
source_vhd_path, source_data_vhd_paths = self._resolve_vhd_sources(runbook)
613628
vhd_path = get_deployable_storage_path(
614-
platform, runbook.vhd, image_location, self._log
615-
)
616-
vhd_details = get_vhd_details(platform, vhd_path)
617-
check_blob_exist(
618-
platform=platform,
619-
account_name=vhd_details["account_name"],
620-
container_name=vhd_details["container_name"],
621-
resource_group_name=vhd_details["resource_group_name"],
622-
blob_name=vhd_details["blob_name"],
623-
raise_error=True,
629+
platform, source_vhd_path, image_location, self._log
624630
)
631+
vhd_details = self._check_blob_exists(platform, vhd_path)
632+
633+
data_vhd_paths: List[Dict[str, Any]] = []
634+
for source_data_vhd in source_data_vhd_paths:
635+
source_data_vhd_path = source_data_vhd["url"]
636+
data_vhd_path = get_deployable_storage_path(
637+
platform, source_data_vhd_path, image_location, self._log
638+
)
639+
self._check_blob_exists(platform, data_vhd_path)
640+
data_vhd: Dict[str, Any] = {"url": data_vhd_path}
641+
if "lun" in source_data_vhd:
642+
data_vhd["lun"] = source_data_vhd["lun"]
643+
data_vhd_paths.append(data_vhd)
625644

626645
# Get features from marketplace image if specified
627646
features = self._get_image_features(platform, runbook.marketplace_source)
@@ -696,6 +715,7 @@ def _internal_run(self) -> Dict[str, Any]:
696715
vhd_details["resource_group_name"],
697716
vhd_details["account_name"],
698717
runbook.gallery_image_location,
718+
data_vhd_paths=data_vhd_paths,
699719
)
700720

701721
sig_url = (
@@ -707,6 +727,75 @@ def _internal_run(self) -> Dict[str, Any]:
707727
self._log.info(f"SIG Url: {sig_url}")
708728
return {self.__sig_name: sig_url}
709729

730+
def _check_blob_exists(
731+
self, platform: AzurePlatform, vhd_path: str
732+
) -> Dict[str, str]:
733+
vhd_details = cast(Dict[str, str], get_vhd_details(platform, vhd_path))
734+
check_blob_exist(
735+
platform=platform,
736+
account_name=vhd_details["account_name"],
737+
container_name=vhd_details["container_name"],
738+
resource_group_name=vhd_details["resource_group_name"],
739+
blob_name=vhd_details["blob_name"],
740+
raise_error=True,
741+
)
742+
return vhd_details
743+
744+
def _resolve_vhd_sources(
745+
self, runbook: SigTransformerSchema
746+
) -> tuple[str, List[Dict[str, Any]]]:
747+
data_vhd_paths: List[Dict[str, Any]] = []
748+
749+
def _add_data_vhd(url: str, lun: Optional[int] = None) -> None:
750+
normalized_url = url.strip()
751+
if not normalized_url:
752+
return
753+
add_secret(normalized_url, PATTERN_URL)
754+
item: Dict[str, Any] = {"url": normalized_url}
755+
if lun is not None:
756+
item["lun"] = lun
757+
data_vhd_paths.append(item)
758+
759+
def _parse_lun(raw_lun: Any) -> Optional[int]:
760+
if isinstance(raw_lun, int) and raw_lun >= 0:
761+
return raw_lun
762+
if isinstance(raw_lun, str):
763+
raw_lun = raw_lun.strip()
764+
if raw_lun.isdigit():
765+
return int(raw_lun)
766+
return None
767+
768+
if isinstance(runbook.vhd, str):
769+
vhd_path = runbook.vhd
770+
elif isinstance(runbook.vhd, dict):
771+
raw_vhd_path = runbook.vhd.get("vhd_path", "")
772+
vhd_path = raw_vhd_path.strip() if isinstance(raw_vhd_path, str) else ""
773+
774+
raw_data_vhd_paths = runbook.vhd.get("data_vhd_paths")
775+
if isinstance(raw_data_vhd_paths, list):
776+
for item in raw_data_vhd_paths:
777+
if not isinstance(item, dict):
778+
continue
779+
780+
# Supported format:
781+
# - data_vhd:
782+
# lun: 0
783+
# url: "https://.../data0.vhd"
784+
raw_data_vhd = item.get("data_vhd")
785+
if isinstance(raw_data_vhd, dict):
786+
url = raw_data_vhd.get("url")
787+
if isinstance(url, str) and url.strip():
788+
_add_data_vhd(url, _parse_lun(raw_data_vhd.get("lun")))
789+
else:
790+
raise LisaException(
791+
f"unsupported type for transformer vhd: {type(runbook.vhd)}"
792+
)
793+
794+
if not vhd_path:
795+
raise LisaException("vhd or vhd.vhd_path must not be empty.")
796+
797+
return vhd_path, data_vhd_paths
798+
710799
def _get_image_features(
711800
self, platform: AzurePlatform, marketplace: str
712801
) -> Dict[str, Any]:

0 commit comments

Comments
 (0)