[Server-cvs] admin/monitor smonplin.cpp, 1.16, 1.17 smonplin.h, 1.8, 1.9
atin at helixcommunity.org atin at helixcommunity.orgUpdate of /cvsroot/server/admin/monitor
In directory cvs01.internal.helixcommunity.org:/tmp/cvs-serv27318/server/admin/monitor
Modified Files:
smonplin.cpp smonplin.h
Log Message:
Synopsis
========
the fact that snmpplin, smonplin and tmplgpln were executing a lot of
dispatchQ callbacks was causing the broadcast receiver to perform poorly due
to the global lock contention.
Branches: SERVER_11_1, SERVER_CURRENT
Reviewed by: dcollins
Description
===========
the helix server is designed so that all threads in the server that have a
mainloop require to acquire the global lock during each iteration and the
dispatchq executes callbacks under the global lock.
dispatchq is the server's inter-process (thread) communication mechanism
whereby thread A acquires thread B's dispatchq lock and puts a SimpleCallback
in its dispatchQ. then thread B acquires the global lock and executes the
SimpleCallback::func(Process*) that thread A had sent.
from this it should b obvious that if the SimpleCallback::func() takes a bit
of time to execute then that means that the global lock is held for that much
time and consequently all other threads who want to acquire the global lock
have to wait. this degrades the server's performance because it essentially
delays each thread's mainloop iteration.
a low risk workaround for this problem was to identify the plugins which lock
the global mutex the longest due to their dispatchq usage and reduce the
amount of work done within the SimpleCallback::func() methods for each of
them. so basically the work that was being done inside the
SimpleCallback::func() method was transferred to a IHXCallback which gets
created inside the SimpleCallback::func() and then it is scheduled to be
executed 1ms later inside the IHXThreadSafeScheduler (without holding the
global lock) within the thread.
Files Affected
==============
server/log/tmplgpln/base_log.cpp
server/log/tmplgpln/base_log.h
server/log/tmplgpln/clientstats_log.cpp
server/log/tmplgpln/clientstats_log.h
server/log/tmplgpln/error_log.cpp
server/log/tmplgpln/error_log.h
server/log/tmplgpln/iLogTemplate.h
server/log/tmplgpln/interval_log.cpp
server/log/tmplgpln/interval_log.h
server/log/tmplgpln/legacy_access_log.cpp
server/log/tmplgpln/legacy_access_log.h
server/log/tmplgpln/outputs.cpp
server/log/tmplgpln/outputs.h
server/log/tmplgpln/rtspevents_log.cpp
server/log/tmplgpln/rtspevents_log.h
server/log/tmplgpln/tmplgpln.cpp
server/log/tmplgpln/tmplgpln.h
server/log/tmplgpln/wildcard_log.cpp
server/log/tmplgpln/wildcard_log.h
server/admin/monitor/smonplin.cpp
server/admin/monitor/smonplin.h
server_rn/snmp/snmpplin/rssnmppln.cpp
server_rn/snmp/snmpplin/rssnmppln.h
server_rn/snmp/snmpplin/snmpplin.exp
Testing Performed
=================
Unit Tests:
N/A
Integration Tests:
(1) load tests were done using 16 inbound streams and 240 clients requesting
the live streams and 300 clients requesting a on-demand clip. the test
ran for 16+ hours and no CAs were seen.
(2) uptime test was run with the modified plugins and 1 HBF was seen due
to contention in registry lock, but the server recovered without
incident and it has been running for 18+ hours with an increased load
capacity of around a 900+ players compared to the previous 700+
players.
Leak Tests:
--lct 1:1:8 was done with 16 inbound streams and 300+ clients
no leaks due to this change were detected.
Performance Tests:
N/A
Platforms Tested: linux-rhel4-i686, sunos-5.8-sparc-server
Build verified: linux-rhel4-i686, sunos-5.8-sparc-server
Index: smonplin.cpp
===================================================================
RCS file: /cvsroot/server/admin/monitor/smonplin.cpp,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- smonplin.cpp 10 Jan 2007 17:07:39 -0000 1.16
+++ smonplin.cpp 29 Jan 2007 23:57:53 -0000 1.17
@@ -118,6 +118,7 @@
, m_pListenSock(NULL)
, m_pRegistry(NULL)
, m_pPropWatch(NULL)
+ , m_pTSSked(NULL)
, m_pCommonClassFactory(NULL)
, m_pMonitorList(NULL)
, m_state(SM_Ready)
@@ -288,6 +289,12 @@
return HXR_UNEXPECTED;
}
+ if (HXR_OK != m_pContext->QueryInterface(IID_IHXThreadSafeScheduler,
+ (void**)&m_pTSSked))
+ {
+ return HXR_UNEXPECTED;
+ }
+
m_pPropWatch->Init((IHXPropWatchResponse *)this);
// initialize the properties we are interested in
@@ -491,73 +498,10 @@
STDMETHODIMP
CSysMon::AddedProp(const UINT32 id, const HXPropType t, const UINT32 parent_id)
{
- HX_RESULT hr = HXR_OK;
- ConnInfo* pClientInfo = NULL;
- WatchProp* pClientWatch = NULL;
-
- // The Player Count was updated
- if (parent_id == m_ServerPropList[0].id && // "server"
- id == m_ServerPropList[1].id) // "server.playercount"
- {
- // retrieve the property value
- if (HXR_OK != m_pRegistry->GetIntById(id, m_Counters.lPlayers))
- {
- hr = HXR_UNEXPECTED;
- }
- m_Counters.lMaxPlayers = m_Counters.lPlayers;
- }
- // The Encoder Count was updated
- else if (parent_id == m_EncoderPropList[0].id && // "LiveConnections"
- id == m_EncoderPropList[1].id) // "LiveConnections.count"
- {
- // retrieve the property value
- if (HXR_OK != m_pRegistry->GetIntById(id, m_Counters.lEncoders))
- {
- hr = HXR_UNEXPECTED;
- }
- }
- // The Splitter Count was updated
- else if (parent_id == m_SplitterPropList[0].id && // "Splitters"
- id == m_SplitterPropList[1].id) // "Splitters.count"
- {
- // retrieve the property value
- if (HXR_OK != m_pRegistry->GetIntById(id, m_Counters.lSplitters))
- {
- hr = HXR_UNEXPECTED;
- }
- }
- // A new LiveConnection was added to the registry
- else if (parent_id == m_EncoderPropList[0].id) // "LiveConnections"
- {
- hr = HandleNewEncoder(id, t, parent_id);
- }
- // A new LiveConnections.Entry#.xxx was added to the registry
- else if ((pClientInfo = FindClient(parent_id)) &&
- (pClientInfo->nType == RTTPE_ENCODER))
- {
- hr = HandleNewEncoderChild(id, t, parent_id, pClientInfo);
- }
- // A new Splitter was added to the registry
- else if (parent_id == m_SplitterPropList[0].id) // "Splitters"
- {
- hr = HandleNewSplitter(id, t, parent_id);
- }
- // A new Splitters.Entry#.Session.# was added to the registry
- else if ((pClientWatch = FindClientWatch(parent_id)) &&
- (pClientWatch->nType == RTTPE_SPLITTER_PLAYER))
- {
- hr = HandleNewSplitterSession(id, t, parent_id, pClientWatch);
- }
- // A new Splitters.Entry#.Session.#.xxx was added to the registry
- else if ((pClientInfo = FindClient(parent_id)) &&
- ((pClientInfo->nType == RTTPE_PLAYER) ||
- (pClientInfo->nType == RTTPE_BROWSER)))
- {
- hr = HandleNewSplitterSessionChild(id, t, parent_id, pClientInfo);
- }
-
-
- return hr;
+ PropWatchResponseTSCallback* pAddedPropCb =
+ new PropWatchResponseTSCallback('a', this, id, t, parent_id, m_pRegistry);
+ m_pTSSked->RelativeEnter(pAddedPropCb, 1); // 1ms later
+ return HXR_OK;
}
HX_RESULT
@@ -845,110 +789,19 @@
STDMETHODIMP
CSysMon::ModifiedProp(const UINT32 id, const HXPropType t, const UINT32 parent_id)
{
- HX_RESULT hr = HXR_OK;
-
- if (id == m_ServerPropList[1].id) // server.clientcount
- {
- // retrieve the property value(int)
- m_pRegistry->GetIntById(id, m_Counters.lPlayers);
-
- (m_Counters.lMaxPlayers < m_Counters.lPlayers)?(m_Counters.lMaxPlayers = m_Counters.lPlayers):0;
- }
- else if (id == m_EncoderPropList[1].id) // LiveConnections.count
- {
- // retrieve the property value(int)
- m_pRegistry->GetIntById(id, m_Counters.lEncoders);
- }
- else if (id == m_SplitterPropList[1].id) // Splitters.count
- {
- // retrieve the property value(int)
- m_pRegistry->GetIntById(id, m_Counters.lSplitters);
- }
-
- return hr;
+ PropWatchResponseTSCallback* pModifiedPropCb =
+ new PropWatchResponseTSCallback('m', this, id, t, parent_id, m_pRegistry);
+ m_pTSSked->RelativeEnter(pModifiedPropCb, 1); // 1ms later
+ return HXR_OK;
}
STDMETHODIMP
CSysMon::DeletedProp(const UINT32 id, const UINT32 parent_id)
{
- WatchProp* pClientWatch = NULL;
- ConnInfo* pClientInfo = NULL;
- ConnInfo* pEncoderInfo = NULL;
- LISTPOSITION pos = 0;
-
- if (parent_id == m_SplitterPropList[0].id)
- {
- pos = m_pClientWatchList->GetHeadPosition();
- while(pos)
- {
- pClientWatch = (WatchProp*)m_pClientWatchList->GetAt(pos);
-
- if((UINT32)pClientWatch->ulClientID == id)
- {
- m_pClientWatchList->RemoveAt(pos);
-
- m_ulSplitterParentID = pClientWatch->id;
-
- HX_DELETE(pClientWatch);
-
- // Restart scan from the beginning
- pos = m_pClientWatchList->GetHeadPosition();
- }
- else
- {
- m_pClientWatchList->GetNext(pos);
- }
- }
-
- }
-
- // Encoder ("LiveConnections")
- else if (parent_id == m_EncoderPropList[0].id)
- {
- pos = m_pEncoderList->GetHeadPosition();
- while(pos)
- {
- pEncoderInfo = (ConnInfo*)m_pEncoderList->GetAt(pos);
-
- if((UINT32)pEncoderInfo->ulRegID == id)
- {
- m_pEncoderList->RemoveAt(pos);
-
- NotifyMonitors(pEncoderInfo, ACTION_Removed);
-
- HX_DELETE(pEncoderInfo);
- break;
- }
-
- m_pEncoderList->GetNext(pos);
- }
- }
-
- // Encoder ("Splitters")
- else if (parent_id == m_ulSplitterParentID)
- {
- pos = m_pSplitterList->GetHeadPosition();
- while(pos)
- {
- pClientInfo = (ConnInfo*)m_pSplitterList->GetAt(pos);
-
- if((UINT32)pClientInfo->ulRegID == id)
- {
- m_pSplitterList->RemoveAt(pos);
-
- NotifyMonitors(pClientInfo, ACTION_Removed);
-
- m_ulSplitterParentID = 0;
-
- HX_DELETE(pClientInfo);
- break;
- }
-
- m_pSplitterList->GetNext(pos);
- }
- }
-
+ PropWatchResponseTSCallback* pDeletedPropCb =
+ new PropWatchResponseTSCallback('d', this, id, (HXPropType)0, parent_id, m_pRegistry);
+ m_pTSSked->RelativeEnter(pDeletedPropCb, 1); // 1ms later
return HXR_OK;
}
@@ -2022,3 +1875,266 @@
{
return 0;
}
+
+CSysMon::PropWatchResponseTSCallback::PropWatchResponseTSCallback(
+ int cType,
+ CSysMon* pSysMon,
+ const UINT32 ulRegPropId,
+ const HXPropType eRegPropType,
+ const UINT32 ulRegPropParentId,
+ IHXRegistry* pRegistry)
+ : m_cType(cType)
+ , m_pSysMon(pSysMon)
+ , m_lRefCount(0)
+ , m_ulRegPropId(ulRegPropId)
+ , m_eRegPropType(eRegPropType)
+ , m_ulRegPropParentId(ulRegPropParentId)
+ , m_pRegistry(pRegistry)
+{
+ if (m_pSysMon)
+ m_pSysMon->AddRef();
+}
+
+CSysMon::PropWatchResponseTSCallback::~PropWatchResponseTSCallback()
+{
+ HX_RELEASE(m_pSysMon);
+}
+
+
+/////////////////////////////////////////////////////////////////////////
+// Method:
+// IUnknown::AddRef
+// Purpose:
+// Everyone usually implements this the same... feel free to use
+// this implementation.
+//
+STDMETHODIMP_(ULONG32) CSysMon::PropWatchResponseTSCallback::AddRef()
+{
+ return InterlockedIncrement(&m_lRefCount);
+}
+
+/////////////////////////////////////////////////////////////////////////
+// Method:
+// IUnknown::Release
+// Purpose:
+// Everyone usually implements this the same... feel free to use
+// this implementation.
+//
+STDMETHODIMP_(ULONG32) CSysMon::PropWatchResponseTSCallback::Release()
+{
+ if (InterlockedDecrement(&m_lRefCount) > 0)
+ {
+ return m_lRefCount;
+ }
+
+ delete this;
+ return 0;
+}
+
+/////////////////////////////////////////////////////////////////////////
+// Method:
+// IUnknown::QueryInterface
+// Purpose:
+// Implement this to export the interfaces supported by your
+// object.
+//
+STDMETHODIMP CSysMon::PropWatchResponseTSCallback::QueryInterface(REFIID riid, void** ppvObj)
+{
+ if (IsEqualIID(riid, IID_IUnknown))
+ {
+ AddRef();
+ *ppvObj = this;
+ return HXR_OK;
+ }
+ else if (IsEqualIID(riid, IID_IHXCallback))
+ {
+ AddRef();
+ *ppvObj = (IHXCallback*)this;
+ return HXR_OK;
+ }
+
+ *ppvObj = NULL;
+ return HXR_NOINTERFACE;
+}
+
+STDMETHODIMP
+CSysMon::PropWatchResponseTSCallback::Func(void)
+{
+ if (m_cType == 'a')
+ {
+ HX_RESULT hr = HXR_OK;
+ ConnInfo* pClientInfo = NULL;
+ WatchProp* pClientWatch = NULL;
+
+ // The Player Count was updated
+ if (m_ulRegPropParentId == m_pSysMon->m_ServerPropList[0].id && // "server"
+ m_ulRegPropId == m_pSysMon->m_ServerPropList[1].id) // "server.playercount"
+ {
+ // retrieve the property value
+ if (HXR_OK != m_pRegistry->GetIntById(m_ulRegPropId, m_pSysMon->m_Counters.lPlayers))
+ {
+ hr = HXR_UNEXPECTED;
+ }
+ m_pSysMon->m_Counters.lMaxPlayers = m_pSysMon->m_Counters.lPlayers;
+ }
+ // The Encoder Count was updated
+ else if (m_ulRegPropParentId == m_pSysMon->m_EncoderPropList[0].id && // "LiveConnections"
+ m_ulRegPropId == m_pSysMon->m_EncoderPropList[1].id) // "LiveConnections.count"
+ {
+ // retrieve the property value
+ if (HXR_OK != m_pRegistry->GetIntById(m_ulRegPropId, m_pSysMon->m_Counters.lEncoders))
+ {
+ hr = HXR_UNEXPECTED;
+ }
+ }
+ // The Splitter Count was updated
+ else if (m_ulRegPropParentId == m_pSysMon->m_SplitterPropList[0].id && // "Splitters"
+ m_ulRegPropId == m_pSysMon->m_SplitterPropList[1].id) // "Splitters.count"
+ {
+ // retrieve the property value
+ if (HXR_OK != m_pRegistry->GetIntById(m_ulRegPropId, m_pSysMon->m_Counters.lSplitters))
+ {
+ hr = HXR_UNEXPECTED;
+ }
+ }
+ // A new LiveConnection was added to the registry
+ else if (m_ulRegPropParentId == m_pSysMon->m_EncoderPropList[0].id) // "LiveConnections"
+ {
+ hr = m_pSysMon->HandleNewEncoder(m_ulRegPropId, m_eRegPropType, m_ulRegPropParentId);
+ }
+ // A new LiveConnections.Entry#.xxx was added to the registry
+ else if ((pClientInfo = m_pSysMon->FindClient(m_ulRegPropParentId)) &&
+ (pClientInfo->nType == RTTPE_ENCODER))
+ {
+ hr = m_pSysMon->HandleNewEncoderChild(m_ulRegPropId, m_eRegPropType, m_ulRegPropParentId, pClientInfo);
+ }
+ // A new Splitter was added to the registry
+ else if (m_ulRegPropParentId == m_pSysMon->m_SplitterPropList[0].id) // "Splitters"
+ {
+ hr = m_pSysMon->HandleNewSplitter(m_ulRegPropId, m_eRegPropType, m_ulRegPropParentId);
+ }
+ // A new Splitters.Entry#.Session.# was added to the registry
+ else if ((pClientWatch = m_pSysMon->FindClientWatch(m_ulRegPropParentId)) &&
+ (pClientWatch->nType == RTTPE_SPLITTER_PLAYER))
+ {
+ hr = m_pSysMon->HandleNewSplitterSession(m_ulRegPropId, m_eRegPropType, m_ulRegPropParentId, pClientWatch);
+ }
+ // A new Splitters.Entry#.Session.#.xxx was added to the registry
+ else if ((pClientInfo = m_pSysMon->FindClient(m_ulRegPropParentId)) &&
+ ((pClientInfo->nType == RTTPE_PLAYER) ||
+ (pClientInfo->nType == RTTPE_BROWSER)))
+ {
+ hr = m_pSysMon->HandleNewSplitterSessionChild(m_ulRegPropId, m_eRegPropType, m_ulRegPropParentId, pClientInfo);
+ }
+
+ return hr;
+ }
+ else if (m_cType == 'm')
+ {
+ HX_RESULT hr = HXR_OK;
+
+ if (m_ulRegPropId == m_pSysMon->m_ServerPropList[1].id) // server.clientcount
+ {
+ // retrieve the property value(int)
+ m_pRegistry->GetIntById(m_ulRegPropId, m_pSysMon->m_Counters.lPlayers);
+
+ (m_pSysMon->m_Counters.lMaxPlayers < m_pSysMon->m_Counters.lPlayers)
+ ? (m_pSysMon->m_Counters.lMaxPlayers = m_pSysMon->m_Counters.lPlayers) : 0;
+ }
+ else if (m_ulRegPropId == m_pSysMon->m_EncoderPropList[1].id) // LiveConnections.count
+ {
+ // retrieve the property value(int)
+ m_pRegistry->GetIntById(m_ulRegPropId, m_pSysMon->m_Counters.lEncoders);
+ }
+ else if (m_ulRegPropId == m_pSysMon->m_SplitterPropList[1].id) // Splitters.count
+ {
+ // retrieve the property value(int)
+ m_pRegistry->GetIntById(m_ulRegPropId, m_pSysMon->m_Counters.lSplitters);
+ }
+
+ return hr;
+ }
+ else if (m_cType = 'd')
+ {
+ WatchProp* pClientWatch = NULL;
+ ConnInfo* pClientInfo = NULL;
+ ConnInfo* pEncoderInfo = NULL;
+ LISTPOSITION pos = 0;
+
+ if (m_ulRegPropParentId == m_pSysMon->m_SplitterPropList[0].id)
+ {
+ pos = m_pSysMon->m_pClientWatchList->GetHeadPosition();
+ while(pos)
+ {
+ pClientWatch = (WatchProp*)(m_pSysMon->m_pClientWatchList)->GetAt(pos);
+
+ if((UINT32)pClientWatch->ulClientID == m_ulRegPropId)
+ {
+ m_pSysMon->m_pClientWatchList->RemoveAt(pos);
+
+ m_pSysMon->m_ulSplitterParentID = pClientWatch->id;
+
+ HX_DELETE(pClientWatch);
+
+ // Restart scan from the beginning
+ pos = m_pSysMon->m_pClientWatchList->GetHeadPosition();
+ }
+ else
+ {
+ m_pSysMon->m_pClientWatchList->GetNext(pos);
+ }
+ }
+
+ }
+
+ // Encoder ("LiveConnections")
+ else if (m_ulRegPropParentId == m_pSysMon->m_EncoderPropList[0].id)
+ {
+ pos = m_pSysMon->m_pEncoderList->GetHeadPosition();
+ while(pos)
+ {
+ pEncoderInfo = (ConnInfo*)(m_pSysMon->m_pEncoderList)->GetAt(pos);
+
+ if((UINT32)pEncoderInfo->ulRegID == m_ulRegPropId)
+ {
+ m_pSysMon->m_pEncoderList->RemoveAt(pos);
+
+ m_pSysMon->NotifyMonitors(pEncoderInfo, ACTION_Removed);
+
+ HX_DELETE(pEncoderInfo);
+ break;
+ }
+
+ m_pSysMon->m_pEncoderList->GetNext(pos);
+ }
+ }
+
+ // Encoder ("Splitters")
+ else if (m_ulRegPropParentId == m_pSysMon->m_ulSplitterParentID)
+ {
+ pos = m_pSysMon->m_pSplitterList->GetHeadPosition();
+ while(pos)
+ {
+ pClientInfo = (ConnInfo*)(m_pSysMon->m_pSplitterList)->GetAt(pos);
+
+ if((UINT32)pClientInfo->ulRegID == m_ulRegPropId)
+ {
+ m_pSysMon->m_pSplitterList->RemoveAt(pos);
+
+ m_pSysMon->NotifyMonitors(pClientInfo, ACTION_Removed);
+
+ m_pSysMon->m_ulSplitterParentID = 0;
+
+ HX_DELETE(pClientInfo);
+ break;
+ }
+
+ m_pSysMon->m_pSplitterList->GetNext(pos);
+ }
+ }
+
+ return HXR_OK;
+ }
+ return HXR_OK;
+}
+
Index: smonplin.h
===================================================================
RCS file: /cvsroot/server/admin/monitor/smonplin.h,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- smonplin.h 11 Jul 2004 23:46:12 -0000 1.8
+++ smonplin.h 29 Jan 2007 23:57:53 -0000 1.9
@@ -99,6 +99,7 @@
IHXNetServices* m_pNetSvc;
IHXListeningSocket* m_pListenSock;
IHXRegistry* m_pRegistry;
+ IHXThreadSafeScheduler* m_pTSSked;
IHXPropWatch* m_pPropWatch;
IHXCommonClassFactory* m_pCommonClassFactory;
@@ -196,8 +197,43 @@
virtual ~CSysMon();
+ class PropWatchResponseTSCallback : public IHXCallback
+ {
+ public:
+ PropWatchResponseTSCallback(
+ int cType,
+ CSysMon* pSysMon,
+ const UINT32 ulRegPropId,
+ const HXPropType eRegPropType,
+ const UINT32 ulRegPropParentId,
+ IHXRegistry* pRegistry);
+
+ ~PropWatchResponseTSCallback();
+
+ // *** IUnknown methods ***
+ STDMETHOD(QueryInterface) (THIS_
+ REFIID riid,
+ void** ppvObj);
+
+ STDMETHOD_(ULONG32,AddRef) (THIS);
+
+ STDMETHOD_(ULONG32,Release) (THIS);
+
+ STDMETHOD(Func) (THIS);
+ protected:
+ int m_cType;
+ CSysMon* m_pSysMon;
+ LONG32 m_lRefCount;
+
+ const UINT32 m_ulRegPropId;
+ const HXPropType m_eRegPropType;
+ const UINT32 m_ulRegPropParentId;
+ IHXRegistry* m_pRegistry;
+ };
public:
+ friend class PropWatchResponseTSCallback;
+
CHXSimpleList* m_pClientWatchList;
CHXSimpleList* m_pPendingClients;
CHXSimpleList* m_pClientList;