search cancel

Devices show Metric Family Discovery status stuck at "Discovering" in Performance Management

book

Article ID: 203496

calendar_today

Updated On:

Products

CA Infrastructure Management CA Performance Management - Usage and Administration DX NetOps

Issue/Introduction

Many devices are showing the Metric Family Discovery Status column as "Discovering". It stays this way and never clears up.

The following errors are seen in the Data Aggregator Exeption.log when this problem occurs.

Default path for the file is /opt/IMDataAggregator/apache-karaf-<version>/data/log.

First this error would be seen:

2020-11-12 17:31:20,1122020-11-12 17:31:20,112 | ERROR | ExceptionLog | A NEW application exception occurred (Key=476a8f4892556e8d27e0774739e8f548af5b3d6b) : java.lang.ArrayIndexOutOfBoundsException | (ExceptionLogger.java:99) 
java.lang.ArrayIndexOutOfBoundsException
    at java.lang.reflect.Array.set(Native Method)[:1.8.0_222]
    at com.ca.im.item.impl.repository.AppendIfAbsentWriteRequest.getModifiedIncomingValue(AppendIfAbsentWriteRequest.java:59)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.WriteRequest.process(WriteRequest.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.GroupWriteRequest.process(GroupWriteRequest.java:80)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.WriteRequest.verifyAndProcess(WriteRequest.java:257)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.AttributeOperations.processItemWriteRequest(AttributeOperations.java:1154)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.AttributeOperations.writeAttributes(AttributeOperations.java:1010)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.ItemRepositoryImpl.writeAttributes(ItemRepositoryImpl.java:1502)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.TenantItemRepository.writeAttributes(TenantItemRepository.java:355)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.tm.reconciliation.impl.AbstractReconciler.detectChangedAttrs(AbstractReconciler.java:530)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.tm.reconciliation.impl.AbstractReconciler.findChanges(AbstractReconciler.java:517)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.tm.reconciliation.impl.AbstractReconciler.findChangedItems(AbstractReconciler.java:475)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.doLocalReconciliation(ComponentCreatorOnReadListener.java:496)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.reconcileItems(ComponentCreatorOnReadListener.java:383)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.onReadComplete(ComponentCreatorOnReadListener.java:154)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.notifyAttributeReadListener(DelegatingAttributeReadListener.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.processSourceResponse(DelegatingAttributeReadListener.java:103)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.onReadComplete(DelegatingAttributeReadListener.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.dm.normalizer.impl.RawAttributeReadListener.onReadComplete(RawAttributeReadListener.java:308)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.notifyAttributeReadListener(DelegatingAttributeReadListener.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.processSourceResponse(DelegatingAttributeReadListener.java:103)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.onReadComplete(DelegatingAttributeReadListener.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.DistributedItemRepository.processReadResponse(DistributedItemRepository.java:1595)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.DistributedItemRepository.access$300(DistributedItemRepository.java:85)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.DistributedItemRepository$3.onResponse(DistributedItemRepository.java:1406)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.DistributedItemRepository$3.onResponse(DistributedItemRepository.java:1402)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.DelegatingGeneralResponseListener.onResponse(DelegatingGeneralResponseListener.java:36)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.DelegatingGeneralResponseListener.onResponse(DelegatingGeneralResponseListener.java:20)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.AsyncMessenger.notifyResponseListener(AsyncMessenger.java:264)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.AsyncMessenger.access$000(AsyncMessenger.java:42)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.AsyncMessenger$ResponseNotificationManager.processNotification(AsyncMessenger.java:69)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.item.impl.distributed.AsyncMessenger$ResponseNotificationManager.processNotification(AsyncMessenger.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
    at com.ca.im.core.concurrent.QueuedNotificationManager$NotificationClass.run(QueuedNotificationManager.java:272)[167:com.ca.im.common.core.util:20.2.2.RELEASE-321]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)[:1.8.0_222]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)[:1.8.0_222]
    at java.lang.Thread.run(Thread.java:748)[:1.8.0_222]

Then this error would appear repeatedly:

2020-11-12 20:18:11,9442020-11-12 20:18:11,944 | ERROR | ExceptionLog | An existing application exception RECURRED (Key=476a8f4892556e8d27e0774739e8f548af5b3d6b), Recurrence count=2 : java.lang.ArrayIndexOutOfBoundsException | (ExceptionLogger.java:104) 

Environment

Observed in r20.2.2 but may be seen in r20.2.5 and earlier releases

Cause

An ArrayIndexOutOfBoundsException is thrown during the component reconciliation process which causes the component discovery to hang and never complete.

Resolution

Defect DE484975 will resolve this in the r20.2.6 release of DX NetOps Performance Management.

The only known work around to resolve the issue is restarting the Data Aggregator dadaemon service.

Fixed Issues documentation

Note that the restart of the Data Aggregator will clear the hung Discoveries. When run again, they will once again hang. The only solution to allow them to complete is upgrading to r20.2.6 for new code that resolves the discovery hangs.

Additional Information

The error described above is the most common one seen when this problem presents itself. In other situations we may not see the same ArrayIndexOutOfBoundsException error. Instead look for this error in the same Data Aggregator karaf.log file. It's easiest found using the Data Aggregator Item ID for the Device affected. That can be found as the ID value in the devices Details tab in the Data Aggregator Monitored Devices Inventory.

In this case the affected device had Item ID 9489739. Searching the karaf.log for that value returns these errors.

WARN  | FacetBatcher     | 2021-01-14 15:15:27,758 | nagedDeviceResourceDiscoveryImpl | nagedDeviceResourceDiscoveryImpl  771 | .im.aggregator.discovery |       | Unable to retrieve mfInfo for key=MultiKey[9489739, {http://im.ca.com/normalizer}NormalizedAddressInfo]
ERROR | b88672f-thread-2 | 2021-01-14 15:15:28,005 | ExceptionLog | .ca.im.core.util.ExceptionLogger   99 | m.ca.im.common.core.util |       | A NEW application exception occurred (Key=476a8f4892556e8d27e0774739e8f548af5b3d6b) : java.lang.NullPointerException
java.lang.NullPointerException
 at java.lang.reflect.Array.set(Native Method)[:1.8.0_222]
 at com.ca.im.item.impl.repository.AppendIfAbsentWriteRequest.getModifiedIncomingValue(AppendIfAbsentWriteRequest.java:59)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.WriteRequest.process(WriteRequest.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.GroupWriteRequest.process(GroupWriteRequest.java:80)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.WriteRequest.verifyAndProcess(WriteRequest.java:257)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.AttributeOperations.processItemWriteRequest(AttributeOperations.java:1154)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.AttributeOperations.writeAttributes(AttributeOperations.java:1010)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.ItemRepositoryImpl.writeAttributes(ItemRepositoryImpl.java:1502)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.TenantItemRepository.writeAttributes(TenantItemRepository.java:355)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.tm.reconciliation.impl.AbstractReconciler.detectChangedAttrs(AbstractReconciler.java:530)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.reconciliation.impl.AbstractReconciler.findChanges(AbstractReconciler.java:517)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.reconciliation.impl.AbstractReconciler.findChangedItems(AbstractReconciler.java:475)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.doLocalReconciliation(ComponentCreatorOnReadListener.java:496)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.reconcileItems(ComponentCreatorOnReadListener.java:383)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.onReadComplete(ComponentCreatorOnReadListener.java:154)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.notifyAttributeReadListener(DelegatingAttributeReadListener.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.processSourceResponse(DelegatingAttributeReadListener.java:103)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.onReadComplete(DelegatingAttributeReadListener.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.dm.normalizer.impl.RawAttributeReadListener.onReadComplete(RawAttributeReadListener.java:308)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.notifyAttributeReadListener(DelegatingAttributeReadListener.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.processSourceResponse(DelegatingAttributeReadListener.java:103)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.onReadComplete(DelegatingAttributeReadListener.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository.processReadResponse(DistributedItemRepository.java:1595)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository.access$300(DistributedItemRepository.java:85)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository$3.onResponse(DistributedItemRepository.java:1406)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository$3.onResponse(DistributedItemRepository.java:1402)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DelegatingGeneralResponseListener.onResponse(DelegatingGeneralResponseListener.java:36)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DelegatingGeneralResponseListener.onResponse(DelegatingGeneralResponseListener.java:20)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger.notifyResponseListener(AsyncMessenger.java:264)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger.access$000(AsyncMessenger.java:42)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger$ResponseNotificationManager.processNotification(AsyncMessenger.java:69)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger$ResponseNotificationManager.processNotification(AsyncMessenger.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.core.concurrent.QueuedNotificationManager$NotificationClass.run(QueuedNotificationManager.java:272)[167:com.ca.im.common.core.util:20.2.2.RELEASE-321]
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)[:1.8.0_222]
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)[:1.8.0_222]
 at java.lang.Thread.run(Thread.java:748)[:1.8.0_222]
ERROR | b88672f-thread-2 | 2021-01-14 15:15:28,006 | DelegatingAttributeReadListener | .DelegatingAttributeReadListener  291 | .im.item-repository.impl |       | 
java.lang.NullPointerException
 at java.lang.reflect.Array.set(Native Method)[:1.8.0_222]
 at com.ca.im.item.impl.repository.AppendIfAbsentWriteRequest.getModifiedIncomingValue(AppendIfAbsentWriteRequest.java:59)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.WriteRequest.process(WriteRequest.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.GroupWriteRequest.process(GroupWriteRequest.java:80)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.WriteRequest.verifyAndProcess(WriteRequest.java:257)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.AttributeOperations.processItemWriteRequest(AttributeOperations.java:1154)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.AttributeOperations.writeAttributes(AttributeOperations.java:1010)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.ItemRepositoryImpl.writeAttributes(ItemRepositoryImpl.java:1502)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.TenantItemRepository.writeAttributes(TenantItemRepository.java:355)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.tm.reconciliation.impl.AbstractReconciler.detectChangedAttrs(AbstractReconciler.java:530)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.reconciliation.impl.AbstractReconciler.findChanges(AbstractReconciler.java:517)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.reconciliation.impl.AbstractReconciler.findChangedItems(AbstractReconciler.java:475)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.doLocalReconciliation(ComponentCreatorOnReadListener.java:496)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.reconcileItems(ComponentCreatorOnReadListener.java:383)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.tm.component.discovery.impl.ComponentCreatorOnReadListener.onReadComplete(ComponentCreatorOnReadListener.java:154)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.notifyAttributeReadListener(DelegatingAttributeReadListener.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.processSourceResponse(DelegatingAttributeReadListener.java:103)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.onReadComplete(DelegatingAttributeReadListener.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.dm.normalizer.impl.RawAttributeReadListener.onReadComplete(RawAttributeReadListener.java:308)[233:com.ca.im.aggregator.discovery:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.notifyAttributeReadListener(DelegatingAttributeReadListener.java:282)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.processSourceResponse(DelegatingAttributeReadListener.java:103)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.repository.DelegatingAttributeReadListener.onReadComplete(DelegatingAttributeReadListener.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository.processReadResponse(DistributedItemRepository.java:1595)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository.access$300(DistributedItemRepository.java:85)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository$3.onResponse(DistributedItemRepository.java:1406)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DistributedItemRepository$3.onResponse(DistributedItemRepository.java:1402)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DelegatingGeneralResponseListener.onResponse(DelegatingGeneralResponseListener.java:36)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.DelegatingGeneralResponseListener.onResponse(DelegatingGeneralResponseListener.java:20)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger.notifyResponseListener(AsyncMessenger.java:264)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger.access$000(AsyncMessenger.java:42)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger$ResponseNotificationManager.processNotification(AsyncMessenger.java:69)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.item.impl.distributed.AsyncMessenger$ResponseNotificationManager.processNotification(AsyncMessenger.java:58)[204:com.ca.im.item-repository.impl:20.2.2.RELEASE-321]
 at com.ca.im.core.concurrent.QueuedNotificationManager$NotificationClass.run(QueuedNotificationManager.java:272)[167:com.ca.im.common.core.util:20.2.2.RELEASE-321]
 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)[:1.8.0_222]
 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)[:1.8.0_222]
 at java.lang.Thread.run(Thread.java:748)[:1.8.0_222]