sudden performance issue on Tomcat Agent system
Article ID: 219616
Updated On:
Products
Issue/Introduction
Seeing DNS Errors
sun.misc.Unsafe.park(Unsafe.java)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
java.util.concurrent.FutureTask.awaitDone(FutureTask.java:426) <1>
com.wily.introscope.agent.dns.DnsServiceExecutor.getDnsHostNameByIPAddr(DnsServiceExecutor.java:289)
com.wily.introscope.agent.trace.io.AddressHostNameHelper.getHostName(AddressHostNameHelper.java:64)
com.wily.introscope.agent.trace.io.ASocketNameFormatter.INameFormatter_format(ASocketNameFormatter.java:67)
com.wily.introscope.agent.trace.ATracerFactory.customFormat(ATracerFactory.java:287) <2>
com.wily.introscope.agent.trace.nio.SocketConcurrentInvocationCounter.ITracerFactory_allocateTracer(SocketConcurrentInvocationCounter.java:41)
com.wily.introscope.agent.trace.InvocationData.IMethodTracer_startTrace(InvocationData.java:1329)
sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java)
oracle.net.nt.TimeoutSocketChannel.write(TimeoutSocketChannel.java:215)
oracle.net.ns.NIOPacket.writeToSocketChannel(NIOPacket.java:211)
oracle.net.ns.NIONSDataChannel.writeDataToSocketChannel(NIONSDataChannel.java:181) <1>
oracle.jdbc.driver.T4CMAREngineNIO.prepareForReading(T4CMAREngineNIO.java:96) <1>
oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:485) <1>
oracle.jdbc.driver.T4C8Oall.doOALL(T4C8Oall.java:612)
oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:226) <2>
oracle.jdbc.driver.OracleStatement.executeMaybeDescribe(OracleStatement.java:904) <1>
oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3780)
oracle.jdbc.driver.T4CPreparedStatement.executeInternal(T4CPreparedStatement.java:1343)
oracle.jdbc.driver.OraclePreparedStatement.executeQuery(OraclePreparedStatement.java:3822)
oracle.jdbc.driver.OraclePreparedStatementWrapper.executeQuery(OraclePreparedStatementWrapper.java:1165)
org.tranql.connector.jdbc.PreparedStatementHandle.executeQuery(PreparedStatementHandle.java:49)
org.eclipse.stardust.engine.core.persistence.jdbc.DmlManager.reloadAttributesFromRow(DmlManager.java:2523) <1>
org.eclipse.stardust.engine.core.persistence.jdbc.Session.reloadAttribute(Session.java:3971)
org.eclipse.stardust.engine.core.persistence.jdbc.DefaultPersistenceController.reloadAttribute(DefaultPersistenceController.java:449)
org.eclipse.stardust.engine.core.persistence.jdbc.PersistentBean.reloadAttribute(PersistentBean.java:121)
org.eclipse.stardust.engine.core.runtime.audittrail.management.ProcessInstanceUtils.isAbortedStateSafe(ProcessInstanceUtils.java:520) <1>
org.eclipse.stardust.engine.core.runtime.beans.ActivityThread.isInAbortingPiHierarchy(ActivityThread.java:543) <3>
org.eclipse.stardust.engine.core.runtime.beans.ActivityInstanceBean.invokeSubprocess(ActivityInstanceBean.java:1458) <3>
org.eclipse.stardust.engine.core.runtime.beans.ActivityThread.runCurrentActivity(ActivityThread.java:1393) <2>
org.eclipse.stardust.engine.core.runtime.beans.ActivityThreadCarrier$ActivityThreadRunner.execute(ActivityThreadCarrier.java:236)
org.eclipse.stardust.engine.core.runtime.beans.SecurityContextAwareAction$SecurityContextBoundAction.execute(SecurityContextAwareAction.java:85)
org.eclipse.stardust.engine.api.spring.AbstractSpringForkingServiceBean$TxIsolatedActionInvoker.isolate(AbstractSpringForkingServiceBean.java:275)
sun.reflect.GeneratedMethodAccessor162.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.CallingInterceptor.invoke(CallingInterceptor.java:31)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.POJOExceptionHandler.invoke(POJOExceptionHandler.java:32)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.RuntimeExtensionsInterceptor.invoke(RuntimeExtensionsInterceptor.java:36)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.NonInteractiveSecurityContextInterceptor.invoke(NonInteractiveSecurityContextInterceptor.java:47)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.api.spring.SpringSessionInterceptor.doWithDataSource(SpringSessionInterceptor.java:145) <1>
org.eclipse.stardust.engine.api.spring.SpringSessionInterceptor$1.doInConnection(SpringSessionInterceptor.java:90)
org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:352)
org.eclipse.stardust.engine.api.spring.SpringSessionInterceptor.invoke(SpringSessionInterceptor.java:84)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.api.spring.SpringConfigurationInterceptor.invoke(SpringConfigurationInterceptor.java:103)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.PropertyLayerProviderInterceptor.invoke(PropertyLayerProviderInterceptor.java:131)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.DebugInterceptor.invoke(DebugInterceptor.java:60)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.api.spring.SpringTxInterceptor$1.doInTransaction(SpringTxInterceptor.java:95)
org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133)
org.eclipse.stardust.engine.api.spring.SpringTxInterceptor.invoke(SpringTxInterceptor.java:73)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.PartitionInterceptor.invoke(PartitionInterceptor.java:42)
org.eclipse.stardust.engine.api.spring.SpringPartitionInterceptor.invoke(SpringPartitionInterceptor.java:30)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132) <1>
org.eclipse.stardust.engine.core.runtime.beans.InvocationManager.invoke(InvocationManager.java:55)
com.sun.proxy.$Proxy118.isolate(Unknown Source)
org.eclipse.stardust.engine.api.spring.AbstractSpringForkingServiceBean$2.doInTransaction(AbstractSpringForkingServiceBean.java:163)
org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133)
org.eclipse.stardust.engine.api.spring.AbstractSpringForkingServiceBean.isolate(AbstractSpringForkingServiceBean.java:144)
org.eclipse.stardust.engine.spring.integration.jms.threading.SystemQueueMessageHandler$SystemQueueMessageDeliveryAction.execute(SystemQueueMessageHandler.java:84)
sun.reflect.GeneratedMethodAccessor248.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.CallingInterceptor.invoke(CallingInterceptor.java:31)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.MultipleTryInterceptor.invoke(MultipleTryInterceptor.java:55)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.spring.integration.jms.threading.MessageHandlingExceptionInterceptor.invoke(MessageHandlingExceptionInterceptor.java:48)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.api.spring.SpringConfigurationInterceptor.invoke(SpringConfigurationInterceptor.java:103)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.PropertyLayerProviderInterceptor.invoke(PropertyLayerProviderInterceptor.java:131)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132)
org.eclipse.stardust.engine.core.runtime.beans.interceptors.PartitionInterceptor.invoke(PartitionInterceptor.java:42)
org.eclipse.stardust.engine.api.spring.SpringPartitionInterceptor.invoke(SpringPartitionInterceptor.java:30)
org.eclipse.stardust.engine.core.runtime.interceptor.MethodInvocationImpl.proceed(MethodInvocationImpl.java:132) <1>
org.eclipse.stardust.engine.core.runtime.beans.InvocationManager.invoke(InvocationManager.java:55)
com.sun.proxy.$Proxy212.execute(Unknown Source)
org.eclipse.stardust.engine.spring.integration.jms.threading.SystemQueueMessageHandler.onMessage(SystemQueueMessageHandler.java:55)
org.springframework.jms.listener.AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:744) <2>
org.springframework.jms.listener.AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:317) <1>
org.springframework.jms.listener.DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1168) <2>
java.lang.Thread.run(Thread.
Environment
Release : 10.7.0
Component : CA DOI Foundations
Resolution
If suspecting slowness is related to the slow DNS lookup, there are 2 properties to try:
[INFO] [IntroscopeAgent.Properties] introscope.agent.dns.lookup.max.wait.in.milliseconds=200
[INFO] [IntroscopeAgent.Properties] introscope.agent.dns.lookup.type=separateThread
May want it to take more or less time depending on your environment.
Try reduce the lookup wait time first.
If it doesn't help much, you can also disable the DNS lookup (introscope.agent.dns.lookup.type=disabled, a restart is needed).
Feedback
