Skip to content

Commit ea1e0f8

Browse files
sanjanchsanjanch
authored
New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361)
* New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions
1 parent 6ccd6d9 commit ea1e0f8

9 files changed

Lines changed: 509 additions & 2 deletions

‎aci-preupgrade-validation-script.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1593,6 +1593,8 @@ def _icurl_error_handler(imdata):
15931593
if imdata and "error" in imdata[0]:
15941594
if "not found in class" in imdata[0]['error']['attributes']['text']:
15951595
raise OldVerPropNotFound('Your current ACI version does not have requested property')
1596+
elif "Incorrect filter format for" in imdata[0]['error']['attributes']['text']:
1597+
raise OldVerPropNotFound('Your current ACI version does not have requested value for the property in the filter')
15961598
elif "unresolved class for" in imdata[0]['error']['attributes']['text']:
15971599
raise OldVerClassNotFound('Your current ACI version does not have requested class')
15981600
elif "not found" in imdata[0]['error']['attributes']['text']:
@@ -6191,6 +6193,39 @@ def n9k_c9408_model_lem_count_check(tversion, fabric_nodes, **kwargs):
61916193

61926194
return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
61936195

6196+
6197+
@check_wrapper(check_title="APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full)")
6198+
def apic_storage_inode_check(**kwargs):
6199+
result = FAIL_UF
6200+
headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Usage %']
6201+
data = []
6202+
unformatted_headers = ['Fault', 'Fault DN']
6203+
unformatted_data = []
6204+
recommended_action = 'Contact Cisco TAC to remove the files in the mount point to free up space and clear the fault'
6205+
doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-storage-inode-usage'
6206+
dn_regex = node_regex + r'/.+p-\[(?P<mountpoint>.+)\]-f'
6207+
desc_regex = r'is (?P<usage>\d{2,3}%) full for Inodes'
6208+
try:
6209+
faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F4388"),eq(faultInst.code,"F4389"),eq(faultInst.code,"F4390"))')
6210+
except OldVerPropNotFound:
6211+
# Pre 5.2.6 does not have these fault codes.
6212+
return Result(result=NA, msg="cversion does not have fault code F4388, F4389 or F4390.", doc_url=doc_url)
6213+
for faultInst in faultInsts:
6214+
lc = faultInst['faultInst']['attributes']['lc']
6215+
if lc not in ["raised", "soaking"]:
6216+
continue
6217+
fc = faultInst['faultInst']['attributes']['code']
6218+
dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn'])
6219+
desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr'])
6220+
if dn and desc:
6221+
data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), desc.group('usage')])
6222+
else:
6223+
unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn']])
6224+
if not data and not unformatted_data:
6225+
result = PASS
6226+
return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url)
6227+
6228+
61946229
# ---- Script Execution ----
61956230

61966231

@@ -6301,6 +6336,7 @@ class CheckManager:
63016336
fabric_port_down_check,
63026337
equipment_disk_limits_exceeded,
63036338
apic_vmm_inventory_sync_faults_check,
6339+
apic_storage_inode_check,
63046340

63056341
# Configurations
63066342
vpc_paired_switches_check,
@@ -6528,4 +6564,3 @@ def main(_args=None):
65286564
prints(msg)
65296565
log.error(msg, exc_info=True)
65306566
sys.exit(1)
6531-

‎docs/docs/validations.md‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ Items | Faults | This Script
8282
[Fabric Port Status][f19] | F1394: ethpm-if-port-down-fabric | :white_check_mark: | :no_entry_sign:
8383
[Equipment Disk Limits][f20] | F1820: 80% -minor<br>F1821: -major<br>F1822: -critical | :white_check_mark: | :no_entry_sign:
8484
[VMM Inventory Partially Synced][f21] | F0132: comp-ctrlr-operational-issues | :white_check_mark: | :no_entry_sign:
85-
85+
[APIC Storage Inode Usage][f22] | F4388: 75% - 85% -warning<br>F4389: 85% - 90% -major<br>F4390: 90% or more -critical | :white_check_mark: | :no_entry_sign:
8686

8787
[f1]: #apic-disk-space-usage
8888
[f2]: #standby-apic-disk-space-usage
@@ -105,6 +105,7 @@ Items | Faults | This Script
105105
[f19]: #fabric-port-status
106106
[f20]: #equipment-disk-limits
107107
[f21]: #vmm-inventory-partially-synced
108+
[f22]: #apic-storage-inode-usage
108109

109110
### Configuration Checks
110111

@@ -1555,6 +1556,56 @@ EPGs using the `pre-provision` resolution immediacy do not rely on the VMM inven
15551556

15561557
This check returns a `MANUAL` result as there are many reasons for a partial inventory sync to be reported. The goal is to ensure that the VMM inventory sync has fully completed before triggering the APIC upgrade to reduce any chance for unexpected inventory changes to occur.
15571558

1559+
1560+
### APIC Storage Inode Usage
1561+
1562+
If a Cisco APIC is running low on inode capacity for any reason, the Cisco APIC upgrade can fail. The Cisco APIC will raise three different faults depending on inode utilization. If any of these faults are raised on the system, the issue should be resolved prior to performing the upgrade.
1563+
1564+
* **F4388**: A warning level fault for Cisco APIC storage inode utilization. This is raised when utilization is greater than 75%.
1565+
1566+
* **F4389**: A major level fault for Cisco APIC storage inode utilization. This is raised when utilization is between 85% and 90%.
1567+
1568+
* **F4390**: A critical level fault for Cisco APIC storage inode utilization. This is raised when utilization is greater than 90%.
1569+
1570+
Although the storage space for the filesystem might be adequate we might still see issues with inode usage, this happens when we have more number of files or directories created with lower file sizes.
1571+
1572+
Recommended Action:
1573+
1574+
To recover from this fault, try the following action
1575+
1576+
1. Free up space from affected disk partition .
1577+
2. TAC may be required to analyze and cleanup certain directories due to filesystem permissions. Cleanup of `/` is one such example.
1578+
1579+
!!! example "Fault Example (F4390: Critical fault for APIC Inode Utilisation)"
1580+
```
1581+
moquery -c faultInst -f 'fault.Inst.code=="F4390"'
1582+
Total Objects shown: 1
1583+
1584+
# faultInst
1585+
ack : yes
1586+
alert : no
1587+
cause : equipment-full
1588+
changeSet : available (Old: 19408344, New: 19407972), inodesFree (Old: 263915, New: 263842), inodesUsed (Old: 2357525, New: 2357598),
1589+
used (Old: 19436092, New: 19436464)
1590+
code : F4390
1591+
created : 2024-08-05T05:42:31.975+02:00
1592+
delegated : no
1593+
descr : Storage unit /scratch-writes on node 3 with hostname apic3 mounted at /scratch-writes is 90% full for Inodes
1594+
dn : topology/pod-2/node-3/sys/ch/p-[/scratch-writes]-f-[/dev/mapper/atx-scratch]/fault-F4390
1595+
domain : infra
1596+
highestSeverity : critical
1597+
lastTransition : 2024-08-05T09:41:18.152+02:00
1598+
lc : raised
1599+
occur : 2
1600+
origSeverity : critical
1601+
prevSeverity : cleared
1602+
rule : eqpt-storage-inode-critical
1603+
severity : critical
1604+
subject : equipment-full
1605+
type : operational
1606+
```
1607+
1608+
15581609
## Configuration Check Details
15591610

15601611
### VPC-paired Leaf switches

‎tests/checks/apic_storage_inode_full_check/Fault_combination.json‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
[
2+
{
3+
"faultInst": {
4+
"attributes": {
5+
"ack": "no",
6+
"alert": "no",
7+
"cause": "equipment-full",
8+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
9+
"code": "F4388",
10+
"created": "2026-03-06T11:58:43.579+00:00",
11+
"delegated": "no",
12+
"descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes",
13+
"dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388",
14+
"domain": "infra",
15+
"highestSeverity": "warning",
16+
"lastTransition": "2026-03-06T11:58:43.579+00:00",
17+
"lc": "raised",
18+
"occur": "1",
19+
"origSeverity": "warning",
20+
"prevSeverity": "warning",
21+
"rule": "eqpt-storage-inode-warning",
22+
"severity": "warning",
23+
"subject": "equipment-full",
24+
"type": "operational"
25+
}
26+
}
27+
},
28+
{
29+
"faultInst": {
30+
"attributes": {
31+
"ack": "no",
32+
"alert": "no",
33+
"cause": "equipment-full",
34+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
35+
"code": "F4388",
36+
"created": "2026-03-06T11:58:43.587+00:00",
37+
"delegated": "no",
38+
"descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes",
39+
"dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388",
40+
"domain": "infra",
41+
"highestSeverity": "warning",
42+
"lastTransition": "2026-03-06T11:58:43.587+00:00",
43+
"lc": "soaking",
44+
"occur": "1",
45+
"origSeverity": "warning",
46+
"prevSeverity": "warning",
47+
"rule": "eqpt-storage-inode-warning",
48+
"severity": "warning",
49+
"subject": "equipment-full",
50+
"type": "operational"
51+
}
52+
}
53+
},
54+
{
55+
"faultInst": {
56+
"attributes": {
57+
"ack": "no",
58+
"alert": "no",
59+
"cause": "equipment-full",
60+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
61+
"code": "F4388",
62+
"created": "2026-03-06T11:58:43.595+00:00",
63+
"delegated": "no",
64+
"descr": "Storage unit /scratch-writes on Node 1 mounted at /scratch-writes is 82% full for Inodes",
65+
"dn": "topology/pod-1/node-1/sys/ch/p-[/scratch-writes]-f-[/dev/mapper/atx-scratch]/fault-F4388",
66+
"domain": "infra",
67+
"highestSeverity": "warning",
68+
"lastTransition": "2026-03-06T11:58:43.595+00:00",
69+
"lc": "raised-clearing",
70+
"occur": "1",
71+
"origSeverity": "warning",
72+
"prevSeverity": "warning",
73+
"rule": "eqpt-storage-inode-warning",
74+
"severity": "warning",
75+
"subject": "equipment-full",
76+
"type": "operational"
77+
}
78+
}
79+
}
80+
]

‎tests/checks/apic_storage_inode_full_check/Fault_exists_not_raised.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
[
2+
{
3+
"faultInst": {
4+
"attributes": {
5+
"ack": "no",
6+
"alert": "no",
7+
"cause": "equipment-full",
8+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
9+
"code": "F4388",
10+
"created": "2026-03-06T11:58:43.579+00:00",
11+
"delegated": "no",
12+
"descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes",
13+
"dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388",
14+
"domain": "infra",
15+
"highestSeverity": "warning",
16+
"lastTransition": "2026-03-06T11:58:43.579+00:00",
17+
"lc": "cleared",
18+
"occur": "1",
19+
"origSeverity": "warning",
20+
"prevSeverity": "warning",
21+
"rule": "eqpt-storage-inode-warning",
22+
"severity": "warning",
23+
"subject": "equipment-full",
24+
"type": "operational"
25+
}
26+
}
27+
},
28+
{
29+
"faultInst": {
30+
"attributes": {
31+
"ack": "no",
32+
"alert": "no",
33+
"cause": "equipment-full",
34+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
35+
"code": "F4388",
36+
"created": "2026-03-06T11:58:43.587+00:00",
37+
"delegated": "no",
38+
"descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes",
39+
"dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388",
40+
"domain": "infra",
41+
"highestSeverity": "warning",
42+
"lastTransition": "2026-03-06T11:58:43.587+00:00",
43+
"lc": "retaining",
44+
"occur": "1",
45+
"origSeverity": "warning",
46+
"prevSeverity": "warning",
47+
"rule": "eqpt-storage-inode-warning",
48+
"severity": "warning",
49+
"subject": "equipment-full",
50+
"type": "operational"
51+
}
52+
}
53+
}
54+
]

‎tests/checks/apic_storage_inode_full_check/Fault_raised.json‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
[
2+
{
3+
"faultInst": {
4+
"attributes": {
5+
"ack": "no",
6+
"alert": "no",
7+
"cause": "equipment-full",
8+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
9+
"code": "F4388",
10+
"created": "2026-03-06T11:58:43.579+00:00",
11+
"delegated": "no",
12+
"descr": "Storage unit /data/admin/bin/avread on Node 1 mounted at /data/admin/bin/avread is 82% full for Inodes",
13+
"dn": "topology/pod-1/node-1/sys/ch/p-[/data/admin/bin/avread]-f-[overlayfs]/fault-F4388",
14+
"domain": "infra",
15+
"highestSeverity": "warning",
16+
"lastTransition": "2026-03-06T12:00:53.560+00:00",
17+
"lc": "raised",
18+
"occur": "1",
19+
"origSeverity": "warning",
20+
"prevSeverity": "warning",
21+
"rule": "eqpt-storage-inode-warning",
22+
"severity": "warning",
23+
"subject": "equipment-full",
24+
"type": "operational"
25+
}
26+
}
27+
},
28+
{
29+
"faultInst": {
30+
"attributes": {
31+
"ack": "no",
32+
"alert": "no",
33+
"cause": "equipment-full",
34+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
35+
"code": "F4388",
36+
"created": "2026-03-06T11:58:43.587+00:00",
37+
"delegated": "no",
38+
"descr": "Storage unit /etc/hosts on Node 1 mounted at /etc/hosts is 82% full for Inodes",
39+
"dn": "topology/pod-1/node-1/sys/ch/p-[/etc/hosts]-f-[overlayfs]/fault-F4388",
40+
"domain": "infra",
41+
"highestSeverity": "warning",
42+
"lastTransition": "2026-03-06T12:00:53.560+00:00",
43+
"lc": "raised",
44+
"occur": "1",
45+
"origSeverity": "warning",
46+
"prevSeverity": "warning",
47+
"rule": "eqpt-storage-inode-warning",
48+
"severity": "warning",
49+
"subject": "equipment-full",
50+
"type": "operational"
51+
}
52+
}
53+
},
54+
{
55+
"faultInst": {
56+
"attributes": {
57+
"ack": "no",
58+
"alert": "no",
59+
"cause": "equipment-full",
60+
"changeSet": "available (Old: 37868344, New: 37859228), inodesFree (Old: 810163, New: 479339), inodesUsed (Old: 1811277, New: 2142101), inodesUtilized (Old: 70, New: 82), used (Old: 976092, New: 985208)",
61+
"code": "F4388",
62+
"created": "2026-03-06T11:58:43.602+00:00",
63+
"delegated": "no",
64+
"descr": "Storage unit / on Node 1 mounted at / is 82% full for Inodes",
65+
"dn": "topology/pod-1/node-1/sys/ch/p-[/]-f-[overlayfs]/fault-F4388",
66+
"domain": "infra",
67+
"highestSeverity": "warning",
68+
"lastTransition": "2026-03-06T12:00:53.560+00:00",
69+
"lc": "raised",
70+
"occur": "1",
71+
"origSeverity": "warning",
72+
"prevSeverity": "warning",
73+
"rule": "eqpt-storage-inode-warning",
74+
"severity": "warning",
75+
"subject": "equipment-full",
76+
"type": "operational"
77+
}
78+
}
79+
}
80+
]

0 commit comments

Comments
 (0)