Skip to content

Commit dc09913

Browse files
asraf-khanasraf-khan
authored
Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365)
* Add check for CFD - CSCwr66848
1 parent ea1e0f8 commit dc09913

5 files changed

Lines changed: 365 additions & 0 deletions

File tree

‎aci-preupgrade-validation-script.py‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6226,6 +6226,71 @@ def apic_storage_inode_check(**kwargs):
62266226
return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url)
62276227

62286228

6229+
# Connection Based Check
6230+
@check_wrapper(check_title="Multi-Pod Modular Spine Bootscript File")
6231+
def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, password, **kwargs):
6232+
result = PASS
6233+
headers = ["Pod ID", "Node ID", "Node Name", "Model", "Bootscript Present"]
6234+
data = []
6235+
recommended_action = (
6236+
"Change the target version to a fixed version of CSCwr66848. "
6237+
"Or clean reboot these spines without the bootscript file before upgrade."
6238+
)
6239+
doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#multi-pod-modular-spine-bootscript-file"
6240+
6241+
if not tversion:
6242+
return Result(result=MANUAL, msg=TVER_MISSING)
6243+
6244+
if not tversion.same_as("6.1(4h)"):
6245+
return Result(result=NA, msg=VER_NOT_AFFECTED)
6246+
6247+
pod_count_resp = icurl('class', 'fabricSetupP.json?rsp-subtree-include=count')
6248+
if (int(pod_count_resp[0]['moCount']['attributes']['count'])) < 2:
6249+
return Result(result=PASS, msg="Not MultiPod Fabric.")
6250+
6251+
modular_spine_models = {"N9K-C9408", "N9K-C9504", "N9K-C9508", "N9K-C9516"}
6252+
modular_spines = [
6253+
node
6254+
for node in fabric_nodes
6255+
if node["fabricNode"]["attributes"].get("model") in modular_spine_models
6256+
and node["fabricNode"]["attributes"].get("role") == "spine"
6257+
]
6258+
if not modular_spines:
6259+
return Result(result=PASS, msg="No modular spine found in fabric.")
6260+
6261+
has_error = False
6262+
for node in modular_spines:
6263+
attr = node["fabricNode"]["attributes"]
6264+
node_id = attr.get("id")
6265+
node_name = attr.get("name")
6266+
dn = re.search(node_regex, attr.get("dn", ""))
6267+
pod_id = dn.group("pod") if dn else "Unknown"
6268+
tep_ip = attr.get("address")
6269+
model = attr.get("model")
6270+
6271+
c = Connection(tep_ip)
6272+
c.username = username
6273+
c.password = password
6274+
c.log = LOG_FILE
6275+
try:
6276+
c.connect()
6277+
c.cmd("ls -l /bootflash/ | grep boots")
6278+
if "bootscript" not in c.output:
6279+
data.append([pod_id, node_id, node_name, model, "No"])
6280+
except Exception as e:
6281+
ssh_error = "SSH ERROR: {}".format(e)
6282+
data.append([pod_id, node_id, node_name, model, ssh_error])
6283+
has_error = True
6284+
continue
6285+
6286+
if has_error:
6287+
result = ERROR
6288+
elif data:
6289+
result = FAIL_O
6290+
6291+
return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
6292+
6293+
62296294
# ---- Script Execution ----
62306295

62316296

@@ -6404,6 +6469,7 @@ class CheckManager:
64046469

64056470
# Bugs
64066471
observer_db_size_check,
6472+
multipod_modular_spine_bootscript_check,
64076473
]
64086474
cli_checks = [
64096475
# General

‎docs/docs/validations.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ Items | Defect | This Script
197197
[Auto Firmware Update on Switch Discovery][d29] | CSCwe83941 | :white_check_mark: | :no_entry_sign:
198198
[Rogue EP Exception List missing on switches][d30] | CSCwp64296 | :white_check_mark: | :no_entry_sign:
199199
[N9K-C9408 with more than 5 N9K-X9400-16W LEMs][d31] | CSCws82819 | :white_check_mark: | :no_entry_sign:
200+
[Multi-Pod Modular Spine Bootscript File][d32] | CSCwr66848 | :white_check_mark: | :no_entry_sign:
200201

201202
[d1]: #ep-announce-compatibility
202203
[d2]: #eventmgr-db-size-defect-susceptibility
@@ -229,6 +230,7 @@ Items | Defect | This Script
229230
[d29]: #auto-firmware-update-on-switch-discovery
230231
[d30]: #rogue-ep-exception-list-missing-on-switches
231232
[d31]: #n9k-c9408-with-more-than-5-n9k-x9400-16w-lems
233+
[d32]: #multi-pod-modular-spine-bootscript-file
232234

233235
## General Check Details
234236

@@ -2742,6 +2744,15 @@ Due to defect [CSCws82819][65], N9K-C9408 switch will experience a boot loop wit
27422744
To avoid this issue, please upgrade to fix version or use less than 6 N9K-X9400-16W in one chassis.
27432745

27442746

2747+
### Multi-Pod Modular Spine Bootscript File
2748+
2749+
Due to [CSCwr66848][66], in Multi-Pod environments, upgrading a modular spine to 6.1(4h) may result in inter-pod traffic to stop working if the `/bootflash/bootscript` file is missing on the spine prior to the upgrade. The traffic interruption occurs because the spine incorrectly indentifies the reason of its reload, leading to an unnecessary attempt to load the missing bootscript file.
2750+
2751+
This issue happens only when the target version is specifically 6.1(4h).
2752+
2753+
To avoid this issue, change the target version to another version. Or verify that the `bootscript` file exists in the bootflash of each modular spine switch prior to upgrading to 6.1(4h). If the file is missing, you have to do clean reboot on the impacted spine to ensure that `/bootflash/bootscript` gets created again. In case you already upgraded your spine and you are experiencing the traffic impact due to this issue, clean reboot of the spine will restore the traffic.
2754+
2755+
27452756
[0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
27462757
[1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
27472758
[2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
@@ -2808,3 +2819,4 @@ To avoid this issue, please upgrade to fix version or use less than 6 N9K-X9400-
28082819
[63]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/m-auto-firmware-update.html
28092820
[64]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp64296
28102821
[65]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws82819
2822+
[66]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwr66848

‎tests/checks/modular_spine_bootscript_check/fabricNode_with_fixed_spine.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
[
2+
{
3+
"fabricNode": {
4+
"attributes": {
5+
"address": "10.0.0.1",
6+
"dn": "topology/pod-1/node-1",
7+
"fabricSt": "commissioned",
8+
"id": "1",
9+
"model": "APIC-SERVER-L2",
10+
"monPolDn": "uni/fabric/monfab-default",
11+
"name": "apic1",
12+
"nodeType": "unspecified",
13+
"role": "controller"
14+
}
15+
}
16+
},
17+
{
18+
"fabricNode": {
19+
"attributes": {
20+
"address": "10.0.0.201",
21+
"dn": "topology/pod-1/node-201",
22+
"fabricSt": "active",
23+
"id": "201",
24+
"model": "N9K-C9316D-GX2",
25+
"monPolDn": "uni/fabric/monfab-default",
26+
"name": "spine201",
27+
"nodeType": "unspecified",
28+
"role": "spine"
29+
}
30+
}
31+
},
32+
{
33+
"fabricNode": {
34+
"attributes": {
35+
"address": "10.0.0.202",
36+
"dn": "topology/pod-2/node-202",
37+
"fabricSt": "active",
38+
"id": "202",
39+
"model": "N9K-C9316D-GX",
40+
"monPolDn": "uni/fabric/monfab-default",
41+
"name": "spine202",
42+
"nodeType": "unspecified",
43+
"role": "spine"
44+
}
45+
}
46+
}
47+
]

‎tests/checks/modular_spine_bootscript_check/fabricNode_with_modular_spine.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
[
2+
{
3+
"fabricNode": {
4+
"attributes": {
5+
"address": "10.0.0.1",
6+
"dn": "topology/pod-1/node-1",
7+
"fabricSt": "commissioned",
8+
"id": "1",
9+
"model": "APIC-SERVER-L2",
10+
"monPolDn": "uni/fabric/monfab-default",
11+
"name": "apic1",
12+
"nodeType": "unspecified",
13+
"role": "controller"
14+
}
15+
}
16+
},
17+
{
18+
"fabricNode": {
19+
"attributes": {
20+
"address": "10.0.0.201",
21+
"dn": "topology/pod-1/node-201",
22+
"fabricSt": "active",
23+
"id": "201",
24+
"model": "N9K-C9504",
25+
"monPolDn": "uni/fabric/monfab-default",
26+
"name": "spine201",
27+
"nodeType": "unspecified",
28+
"role": "spine"
29+
}
30+
}
31+
},
32+
{
33+
"fabricNode": {
34+
"attributes": {
35+
"address": "10.0.0.202",
36+
"dn": "topology/pod-2/node-202",
37+
"fabricSt": "active",
38+
"id": "202",
39+
"model": "N9K-C9508",
40+
"monPolDn": "uni/fabric/monfab-default",
41+
"name": "spine202",
42+
"nodeType": "unspecified",
43+
"role": "spine"
44+
}
45+
}
46+
}
47+
]

0 commit comments

Comments
 (0)