Stuck Thread in Replication

classic Classic list List threaded Threaded
1 message Options
Reply | Threaded
Open this post in threaded view
|  
Report Content as Inappropriate

Stuck Thread in Replication

sarapitola
Hi, we have a Weblogic Cluster that some times (without a pattern) goes into StuckThread issue on replication.

WLS: 12.1.2
JDK: 1.7_80
Enviroment: Exalogic
Replication: Dedicated channel

"[ACTIVE] ExecuteThread: '124' for queue: 'weblogic.kernel.Default (self-tuning)'" daemon prio=10 tid=0x00007f416f6d6800 nid=0x2081 waiting for monitor entry [0x00007f41571b0000]
   java.lang.Thread.State: BLOCKED (on object monitor)
        at weblogic.rjvm.MsgAbbrevJVMConnection.sendMsgNow(MsgAbbrevJVMConnection.java:214)
        - waiting to lock <0x000000063b09ae48> (a weblogic.rjvm.MsgAbbrevJVMConnection$WritingState)
        at weblogic.rjvm.MsgAbbrevJVMConnection.sendMsg(MsgAbbrevJVMConnection.java:142)
        at weblogic.rjvm.ConnectionManager.sendMsg(ConnectionManager.java:619)
        at weblogic.rjvm.RJVMImpl.send(RJVMImpl.java:942)
        at weblogic.rjvm.MsgAbbrevOutputStream.flushAndSend(MsgAbbrevOutputStream.java:400)
        at weblogic.rjvm.MsgAbbrevOutputStream.sendOneWay(MsgAbbrevOutputStream.java:406)
        at weblogic.rjvm.BasicOutboundRequest.sendOneWay(BasicOutboundRequest.java:105)
        at weblogic.rmi.internal.BasicRemoteRef.sendOneway(BasicRemoteRef.java:412)
        at weblogic.rmi.internal.BasicRemoteRef.invoke(BasicRemoteRef.java:296)
        at weblogic.cluster.replication.ReplicationManager_12120_WLStub.updateOneWay(Unknown Source)
        at sun.reflect.GeneratedMethodAccessor375.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at weblogic.cluster.replication.SecureReplicationInvocationHandler$ReplicationServicesInvocationAction.run(SecureReplicationInvocationHandler.java:194)
        at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
        at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
        at weblogic.cluster.replication.SecureReplicationInvocationHandler.invoke(SecureReplicationInvocationHandler.java:164)
        at com.sun.proxy.$Proxy157.updateOneWay(Unknown Source)
        at weblogic.cluster.replication.ReplicationManager.sendUpdateRequestToSecondary(ReplicationManager.java:734)
        at weblogic.cluster.replication.ReplicationManager.updateSecondary(ReplicationManager.java:666)
        at weblogic.servlet.internal.session.ReplicatedSessionData._syncSession(ReplicatedSessionData.java:609)
        - locked <0x000000067bc75b80> (a weblogic.servlet.internal.session.ReplicatedSessionChange)
        at weblogic.servlet.internal.session.ReplicatedSessionData.syncSession(ReplicatedSessionData.java:571)
        at weblogic.servlet.internal.session.ReplicatedSessionContext.sync(ReplicatedSessionContext.java:94)
        - locked <0x000000067bc75a00> (a weblogic.servlet.internal.session.ReplicatedSessionData)
        at weblogic.servlet.internal.ServletRequestImpl$SessionHelper.syncSession(ServletRequestImpl.java:3194)
        at weblogic.servlet.internal.ServletRequestImpl$SessionHelper.syncSessions(ServletRequestImpl.java:3172)
        at weblogic.servlet.internal.ServletRequestImpl$SessionHelper.syncSession(ServletRequestImpl.java:3160)
        at weblogic.servlet.internal.ServletResponseImpl$1.run(ServletResponseImpl.java:1555)
        at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
        at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
        at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
        at weblogic.servlet.internal.ServletResponseImpl.send(ServletResponseImpl.java:1551)
        at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1574)
        at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
        at weblogic.work.ExecuteThread.execute(ExecuteThread.java:312)
        at weblogic.work.ExecuteThread.run(ExecuteThread.java:264)

"[ACTIVE] ExecuteThread: '29' for queue: 'weblogic.kernel.Default (self-tuning)'" daemon prio=10 tid=0x00007f416eb4d000 nid=0x1adf runnable [0x00007f4161dff000]
   java.lang.Thread.State: RUNNABLE
        at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
        at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
        at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
        at sun.nio.ch.IOUtil.write(IOUtil.java:65)
        at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:492)
        - locked <0x000000063b09a380> (a java.lang.Object)
        at java.nio.channels.Channels.writeFullyImpl(Channels.java:78)
        at java.nio.channels.Channels.writeFully(Channels.java:98)
        - locked <0x000000063b09a458> (a java.lang.Object)
        at java.nio.channels.Channels.access$000(Channels.java:61)
        at java.nio.channels.Channels$1.write(Channels.java:174)
        - locked <0x000000063ceef218> (a java.nio.channels.Channels$1)
        at weblogic.socket.SocketMuxer.internalWrite(SocketMuxer.java:1037)
        at weblogic.socket.SocketMuxer.write(SocketMuxer.java:1029)
        at weblogic.rjvm.t3.MuxableSocketT3$T3MsgAbbrevJVMConnection.sendMsg(MuxableSocketT3.java:562)
        at weblogic.rjvm.MsgAbbrevJVMConnection.sendMsgNow(MsgAbbrevJVMConnection.java:233)
        - locked <0x000000063b09ae48> (a weblogic.rjvm.MsgAbbrevJVMConnection$WritingState)
        at weblogic.rjvm.MsgAbbrevJVMConnection.sendMsg(MsgAbbrevJVMConnection.java:142)
        at weblogic.rjvm.ConnectionManager.sendMsg(ConnectionManager.java:619)
        at weblogic.rjvm.RJVMImpl.send(RJVMImpl.java:942)
        at weblogic.rjvm.MsgAbbrevOutputStream.flushAndSend(MsgAbbrevOutputStream.java:400)
        at weblogic.rjvm.MsgAbbrevOutputStream.sendOneWay(MsgAbbrevOutputStream.java:406)
        at weblogic.rjvm.BasicOutboundRequest.sendOneWay(BasicOutboundRequest.java:105)
        at weblogic.rmi.internal.BasicRemoteRef.sendOneway(BasicRemoteRef.java:412)
        at weblogic.rmi.internal.BasicRemoteRef.invoke(BasicRemoteRef.java:296)
        at weblogic.cluster.replication.ReplicationManager_12120_WLStub.updateOneWay(Unknown Source)
        at sun.reflect.GeneratedMethodAccessor375.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at weblogic.cluster.replication.SecureReplicationInvocationHandler$ReplicationServicesInvocationAction.run(SecureReplicationInvocationHandler.java:194)
        at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
        at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
        at weblogic.cluster.replication.SecureReplicationInvocationHandler.invoke(SecureReplicationInvocationHandler.java:164)
        at com.sun.proxy.$Proxy157.updateOneWay(Unknown Source)
        at weblogic.cluster.replication.ReplicationManager.sendUpdateRequestToSecondary(ReplicationManager.java:734)
        at weblogic.cluster.replication.ReplicationManager.updateSecondary(ReplicationManager.java:666)
        at weblogic.servlet.internal.session.ReplicatedSessionData._syncSession(ReplicatedSessionData.java:609)
        - locked <0x0000000674057cc8> (a weblogic.servlet.internal.session.ReplicatedSessionChange)
        at weblogic.servlet.internal.session.ReplicatedSessionData.syncSession(ReplicatedSessionData.java:571)
        at weblogic.servlet.internal.session.ReplicatedSessionContext.sync(ReplicatedSessionContext.java:94)
        - locked <0x00000006740d2638> (a weblogic.servlet.internal.session.ReplicatedSessionData)
        at weblogic.servlet.internal.ServletRequestImpl$SessionHelper.syncSession(ServletRequestImpl.java:3194)
        at weblogic.servlet.internal.ServletRequestImpl$SessionHelper.syncSessions(ServletRequestImpl.java:3172)
        at weblogic.servlet.internal.ServletRequestImpl$SessionHelper.syncSession(ServletRequestImpl.java:3160)
        at weblogic.servlet.internal.ServletResponseImpl$1.run(ServletResponseImpl.java:1555)
        at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
        at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:120)
        at weblogic.servlet.provider.WlsSubjectHandle.run(WlsSubjectHandle.java:57)
        at weblogic.servlet.internal.ServletResponseImpl.send(ServletResponseImpl.java:1551)
        at weblogic.servlet.internal.ServletRequestImpl.run(ServletRequestImpl.java:1574)
        at weblogic.servlet.provider.ContainerSupportProviderImpl$WlsRequestExecutor.run(ContainerSupportProviderImpl.java:254)
        at weblogic.work.ExecuteThread.execute(ExecuteThread.java:312)
        at weblogic.work.ExecuteThread.run(ExecuteThread.java:264)

We already change to NativeMuxer latest PSUs installed.

Any doubt ask. Every help will be good.
Loading...