diff options
author | Colm Donelan <Colm.Donelan@arm.com> | 2019-11-29 09:10:59 +0000 |
---|---|---|
committer | Matteo Martincigh <matteo.martincigh@arm.com> | 2019-11-29 11:37:17 +0000 |
commit | 2ba48d251d0fe4dca214497ac33283106ca0973b (patch) | |
tree | d92ca2462e1853ac116dc8dd26c7c3afac4103c6 | |
parent | e011d20f279e6e67e899bc6930b0266fc357bc1c (diff) | |
download | armnn-2ba48d251d0fe4dca214497ac33283106ca0973b.tar.gz |
IVGCVSW-4118 Fix long unit test execution
* Reduced sleep_for timings
* Removed duplicate SendStreamMetaDataPacket.
* Modified SendCounterPacket::WaitForPacketSent to use wait_for
* Modified SendCounterPacket::Send WaitingForAck to use wait_for
* Added destructor to StreamRedirector.
* Added method SendCounterPacketTests::HasWrittenData
* Restructured many tests in ProfilingTests.
Signed-off-by: Keith Davis <keith.davis@arm.com>
Change-Id: I55c59cac6674ac40a1056a5302a997d5da9e9d91
Signed-off-by: Colm Donelan <Colm.Donelan@arm.com>
-rw-r--r-- | src/profiling/ConnectionAcknowledgedCommandHandler.cpp | 4 | ||||
-rw-r--r-- | src/profiling/ProfilingService.hpp | 4 | ||||
-rw-r--r-- | src/profiling/SendCounterPacket.cpp | 8 | ||||
-rw-r--r-- | src/profiling/SendCounterPacket.hpp | 7 | ||||
-rw-r--r-- | src/profiling/test/ProfilingGuidTest.cpp | 2 | ||||
-rw-r--r-- | src/profiling/test/ProfilingTests.cpp | 769 | ||||
-rw-r--r-- | src/profiling/test/ProfilingTests.hpp | 74 | ||||
-rw-r--r-- | src/profiling/test/SendCounterPacketTests.cpp | 49 | ||||
-rw-r--r-- | src/profiling/test/SendCounterPacketTests.hpp | 12 |
9 files changed, 536 insertions, 393 deletions
diff --git a/src/profiling/ConnectionAcknowledgedCommandHandler.cpp b/src/profiling/ConnectionAcknowledgedCommandHandler.cpp index 630b555b6e..a2a045d1e0 100644 --- a/src/profiling/ConnectionAcknowledgedCommandHandler.cpp +++ b/src/profiling/ConnectionAcknowledgedCommandHandler.cpp @@ -37,9 +37,7 @@ void ConnectionAcknowledgedCommandHandler::operator()(const Packet& packet) // Once a Connection Acknowledged packet has been received, move to the Active state immediately m_StateMachine.TransitionToState(ProfilingState::Active); - - // Send all the packet required for the handshake with the external profiling service - m_SendCounterPacket.SendStreamMetaDataPacket(); + // Send the counter directory packet. m_SendCounterPacket.SendCounterDirectoryPacket(m_CounterDirectory); m_SendTimelinePacket.SendTimelineMessageDirectoryPackage(); diff --git a/src/profiling/ProfilingService.hpp b/src/profiling/ProfilingService.hpp index ee199d55ab..9fc642fe9d 100644 --- a/src/profiling/ProfilingService.hpp +++ b/src/profiling/ProfilingService.hpp @@ -197,9 +197,9 @@ protected: { instance.m_StateMachine.TransitionToState(newState); } - void WaitForPacketSent(ProfilingService& instance) + void WaitForPacketSent(ProfilingService& instance, uint32_t timeout = 1000) { - return instance.m_SendCounterPacket.WaitForPacketSent(); + return instance.m_SendCounterPacket.WaitForPacketSent(timeout); } }; diff --git a/src/profiling/SendCounterPacket.cpp b/src/profiling/SendCounterPacket.cpp index 1ed926b6d7..2d6458316f 100644 --- a/src/profiling/SendCounterPacket.cpp +++ b/src/profiling/SendCounterPacket.cpp @@ -1012,10 +1012,10 @@ void SendCounterPacket::Send(IProfilingConnection& profilingConnection) // Flush the buffer manually to send the packet FlushBuffer(profilingConnection); - // Wait indefinitely until notified otherwise (it could that the profiling state has changed due to the - // connection being acknowledged, or that new data is ready to be sent, or that the send thread is - // being shut down, etc.) - m_WaitCondition.wait(lock); + // Wait for a connection ack from the remote server. We should expect a response within timeout value. + // If not, drop back to the start of the loop and detect somebody closing the thread. Then send the + // StreamMetadata again. + m_WaitCondition.wait_for(lock, std::chrono::milliseconds(m_Timeout)); // Do not flush the buffer again continue; diff --git a/src/profiling/SendCounterPacket.hpp b/src/profiling/SendCounterPacket.hpp index 102cbccc84..42e84323e4 100644 --- a/src/profiling/SendCounterPacket.hpp +++ b/src/profiling/SendCounterPacket.hpp @@ -64,12 +64,11 @@ public: void Stop(bool rethrowSendThreadExceptions = true); bool IsRunning() { return m_IsRunning.load(); } - void WaitForPacketSent() + void WaitForPacketSent(uint32_t timeout = 1000) { std::unique_lock<std::mutex> lock(m_PacketSentWaitMutex); - - // Blocks until notified that at least a packet has been sent - m_PacketSentWaitCondition.wait(lock); + // Blocks until notified that at least a packet has been sent or until timeout expires. + m_PacketSentWaitCondition.wait_for(lock, std::chrono::milliseconds(timeout)); } private: diff --git a/src/profiling/test/ProfilingGuidTest.cpp b/src/profiling/test/ProfilingGuidTest.cpp index cf5941d70a..5782d21cf8 100644 --- a/src/profiling/test/ProfilingGuidTest.cpp +++ b/src/profiling/test/ProfilingGuidTest.cpp @@ -81,7 +81,7 @@ BOOST_AUTO_TEST_CASE(StaticGuidGeneratorCollisionTest) { ProfilingGuidGenerator generator; std::set<uint64_t> guids; - for (int i = 0; i < 1000000; ++i) + for ( int i = 0; i < 100000; ++i ) { std::stringstream ss; ss << i; diff --git a/src/profiling/test/ProfilingTests.cpp b/src/profiling/test/ProfilingTests.cpp index 4c4ec0ad42..73f1c20246 100644 --- a/src/profiling/test/ProfilingTests.cpp +++ b/src/profiling/test/ProfilingTests.cpp @@ -143,53 +143,90 @@ BOOST_AUTO_TEST_CASE(CheckCommandHandler) profilingStateMachine.TransitionToState(ProfilingState::NotConnected); profilingStateMachine.TransitionToState(ProfilingState::WaitingForAck); + // A 1mSec timeout should be enough to slow the command handler thread a little. CommandHandler commandHandler0(1, true, commandHandlerRegistry, packetVersionResolver); + // This should start the command handler thread return the connection ack and put the profiling + // service into active state. commandHandler0.Start(testProfilingConnectionBase); - commandHandler0.Start(testProfilingConnectionBase); + // Try to start the send thread many times, it must only start once commandHandler0.Start(testProfilingConnectionBase); - commandHandler0.Stop(); + // This could take up to 20mSec but we'll check often. + for (int i = 0; i < 10; i++) + { + if (profilingStateMachine.GetCurrentState() == ProfilingState::Active) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } BOOST_CHECK(profilingStateMachine.GetCurrentState() == ProfilingState::Active); + // Close the thread again. + commandHandler0.Stop(); + profilingStateMachine.TransitionToState(ProfilingState::NotConnected); profilingStateMachine.TransitionToState(ProfilingState::WaitingForAck); - // commandHandler1 should give up after one timeout - CommandHandler commandHandler1(10, true, commandHandlerRegistry, packetVersionResolver); - commandHandler1.Start(testProfilingConnectionTimeOutError); + // In this test we'll simulate a timeout without a connection ack packet being received. + // Stop after timeout is set so we expect the command handler to stop almost immediately. + CommandHandler commandHandler1(1, true, commandHandlerRegistry, packetVersionResolver); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + commandHandler1.Start(testProfilingConnectionTimeOutError); + // Wait until we know a timeout exception has been sent at least once. + for (int i = 0; i < 10; i++) + { + if (testProfilingConnectionTimeOutError.ReadCalledCount()) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } + // and leave another short period for the timeout exception to be processed and the loop to break. + std::this_thread::sleep_for(std::chrono::milliseconds(3)); + // The command handler loop should have stopped after the timeout. BOOST_CHECK(!commandHandler1.IsRunning()); - commandHandler1.Stop(); + commandHandler1.Stop(); + // The state machine should never have received the ack so will still be in WaitingForAck. BOOST_CHECK(profilingStateMachine.GetCurrentState() == ProfilingState::WaitingForAck); - // Now commandHandler1 should persist after a timeout + + // Disable stop after timeout and now commandHandler1 should persist after a timeout commandHandler1.SetStopAfterTimeout(false); + // Restart the thread. commandHandler1.Start(testProfilingConnectionTimeOutError); - for (int i = 0; i < 100; i++) + // Wait for at the three timeouts and the ack to be sent. + for (int i = 0; i < 10; i++) { - if (profilingStateMachine.GetCurrentState() == ProfilingState::Active) + if (testProfilingConnectionTimeOutError.ReadCalledCount() > 3) { break; } - - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); } - commandHandler1.Stop(); + // Even after the 3 exceptions the ack packet should have transitioned the command handler to active. BOOST_CHECK(profilingStateMachine.GetCurrentState() == ProfilingState::Active); - CommandHandler commandHandler2(100, false, commandHandlerRegistry, packetVersionResolver); + // A command handler that gets exceptions other than timeouts should keep going. + CommandHandler commandHandler2(1, false, commandHandlerRegistry, packetVersionResolver); commandHandler2.Start(testProfilingConnectionArmnnError); - // commandHandler2 should not stop once it encounters a non timing error - std::this_thread::sleep_for(std::chrono::milliseconds(500)); + // Wait for two exceptions to be thrown. + for (int i = 0; i < 10; i++) + { + if (testProfilingConnectionTimeOutError.ReadCalledCount() >= 2) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + } BOOST_CHECK(commandHandler2.IsRunning()); commandHandler2.Stop(); @@ -591,50 +628,6 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceDisabled) BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); } -BOOST_AUTO_TEST_CASE(CheckProfilingServiceEnabled) -{ - // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output - LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); - - armnn::Runtime::CreationOptions::ExternalProfilingOptions options; - options.m_EnableProfiling = true; - ProfilingService& profilingService = ProfilingService::Instance(); - profilingService.ResetExternalProfilingOptions(options, true); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - profilingService.Update(); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); - - // Redirect the output to a local stream so that we can parse the warning message - std::stringstream ss; - StreamRedirector streamRedirector(std::cout, ss.rdbuf()); - profilingService.Update(); - BOOST_CHECK(boost::contains(ss.str(), "Cannot connect to stream socket: Connection refused")); -} - -BOOST_AUTO_TEST_CASE(CheckProfilingServiceEnabledRuntime) -{ - // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output - LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); - - armnn::Runtime::CreationOptions::ExternalProfilingOptions options; - ProfilingService& profilingService = ProfilingService::Instance(); - profilingService.ResetExternalProfilingOptions(options, true); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - profilingService.Update(); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - options.m_EnableProfiling = true; - profilingService.ResetExternalProfilingOptions(options); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - profilingService.Update(); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); - - // Redirect the output to a local stream so that we can parse the warning message - std::stringstream ss; - StreamRedirector streamRedirector(std::cout, ss.rdbuf()); - profilingService.Update(); - BOOST_CHECK(boost::contains(ss.str(), "Cannot connect to stream socket: Connection refused")); -} - BOOST_AUTO_TEST_CASE(CheckProfilingServiceCounterDirectory) { armnn::Runtime::CreationOptions::ExternalProfilingOptions options; @@ -653,6 +646,9 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceCounterDirectory) BOOST_CHECK(counterDirectory1.GetCounterCount() == 0); profilingService.Update(); BOOST_CHECK(counterDirectory1.GetCounterCount() != 0); + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); } BOOST_AUTO_TEST_CASE(CheckProfilingServiceCounterValues) @@ -682,7 +678,6 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceCounterValues) writers.push_back(std::thread(&ProfilingService::AddCounterValue, profilingServicePtr, counterUid, 10)); writers.push_back(std::thread(&ProfilingService::SubtractCounterValue, profilingServicePtr, counterUid, 5)); } - std::for_each(writers.begin(), writers.end(), mem_fn(&std::thread::join)); uint32_t counterValue = 0; @@ -692,6 +687,9 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceCounterValues) BOOST_CHECK_NO_THROW(profilingService.SetCounterValue(counterUid, 0)); BOOST_CHECK_NO_THROW(counterValue = profilingService.GetCounterValue(counterUid)); BOOST_CHECK(counterValue == 0); + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); } BOOST_AUTO_TEST_CASE(CheckProfilingObjectUids) @@ -2187,86 +2185,6 @@ BOOST_AUTO_TEST_CASE(RequestCounterDirectoryCommandHandlerTest2) BOOST_TEST(header2Word1 == 419); // data length } -BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadConnectionAcknowledgedPacket) -{ - // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output - LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); - - // Swap the profiling connection factory in the profiling service instance with our mock one - SwapProfilingConnectionFactoryHelper helper; - - // Redirect the standard output to a local stream so that we can parse the warning message - std::stringstream ss; - StreamRedirector streamRedirector(std::cout, ss.rdbuf()); - - // Calculate the size of a Stream Metadata packet - std::string processName = GetProcessName().substr(0, 60); - unsigned int processNameSize = processName.empty() ? 0 : boost::numeric_cast<unsigned int>(processName.size()) + 1; - unsigned int streamMetadataPacketsize = 118 + processNameSize; - - // Reset the profiling service to the uninitialized state - armnn::Runtime::CreationOptions::ExternalProfilingOptions options; - options.m_EnableProfiling = true; - ProfilingService& profilingService = ProfilingService::Instance(); - profilingService.ResetExternalProfilingOptions(options, true); - - // Bring the profiling service to the "WaitingForAck" state - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - profilingService.Update(); // Initialize the counter directory - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); - profilingService.Update(); // Create the profiling connection - - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - - // Remove the packets received so far - mockProfilingConnection->Clear(); - - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); - profilingService.Update(); - - // Wait for the Stream Metadata packet to be sent - helper.WaitForProfilingPacketsSent(); - - // Check that the mock profiling connection contains one Stream Metadata packet - const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); - BOOST_TEST(writtenData.size() == 1); - BOOST_TEST(writtenData[0] == streamMetadataPacketsize); - - // Write a valid "Connection Acknowledged" packet into the mock profiling connection, to simulate a valid - // reply from an external profiling service - - // Connection Acknowledged Packet header (word 0, word 1 is always zero): - // 26:31 [6] packet_family: Control Packet Family, value 0b000000 - // 16:25 [10] packet_id: Packet identifier, value 0b0000000001 - // 8:15 [8] reserved: Reserved, value 0b00000000 - // 0:7 [8] reserved: Reserved, value 0b00000000 - uint32_t packetFamily = 0; - uint32_t packetId = 37; // Wrong packet id!!! - uint32_t header = ((packetFamily & 0x0000003F) << 26) | ((packetId & 0x000003FF) << 16); - - // Create the Connection Acknowledged Packet - Packet connectionAcknowledgedPacket(header); - - // Write the packet to the mock profiling connection - mockProfilingConnection->WritePacket(std::move(connectionAcknowledgedPacket)); - - // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that - // the Connection Acknowledged packet gets processed by the profiling service - std::this_thread::sleep_for(std::chrono::seconds(2)); - - // Check that the expected error has occurred and logged to the standard output - BOOST_CHECK(boost::contains(ss.str(), "Functor with requested PacketId=37 and Version=4194304 does not exist")); - - // The Connection Acknowledged Command Handler should not have updated the profiling state - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); - - // Reset the profiling service to stop any running thread - options.m_EnableProfiling = false; - profilingService.ResetExternalProfilingOptions(options, true); -} - BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodConnectionAcknowledgedPacket) { // Swap the profiling connection factory in the profiling service instance with our mock one @@ -2300,12 +2218,24 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodConnectionAcknowledgedPacket) profilingService.Update(); // Start the command handler and the send thread // Wait for the Stream Metadata packet to be sent - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Check that the mock profiling connection contains one Stream Metadata packet const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); - BOOST_TEST(writtenData.size() == 1); - BOOST_TEST(writtenData[0] == streamMetadataPacketsize); + if (writtenData.size() > 1) + { + // If this thread has been blocked for some time a second or more Stream Metadata packet could have been sent. + // In these cases make sure all packet are of length streamMetadataPacketsize + for(uint32_t packetLength : writtenData) + { + BOOST_TEST(packetLength == streamMetadataPacketsize); + } + } + else + { + BOOST_TEST(writtenData.size() == 1); + BOOST_TEST(writtenData[0] == streamMetadataPacketsize); + } // Write a valid "Connection Acknowledged" packet into the mock profiling connection, to simulate a valid // reply from an external profiling service @@ -2325,8 +2255,8 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodConnectionAcknowledgedPacket) // Write the packet to the mock profiling connection mockProfilingConnection->WritePacket(std::move(connectionAcknowledgedPacket)); - // Wait for the Counter Directory packet to be sent - helper.WaitForProfilingPacketsSent(); + // Wait for the counter directory packet to ensure the ConnectionAcknowledgedCommandHandler has run. + helper.WaitForProfilingPacketsSent(mockProfilingConnection, 5000); // The Connection Acknowledged Command Handler should have updated the profiling state accordingly BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); @@ -2336,18 +2266,11 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodConnectionAcknowledgedPacket) profilingService.ResetExternalProfilingOptions(options, true); } -BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadRequestCounterDirectoryPacket) +BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodRequestCounterDirectoryPacket) { - // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output - LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); - // Swap the profiling connection factory in the profiling service instance with our mock one SwapProfilingConnectionFactoryHelper helper; - // Redirect the standard output to a local stream so that we can parse the warning message - std::stringstream ss; - StreamRedirector streamRedirector(std::cout, ss.rdbuf()); - // Reset the profiling service to the uninitialized state armnn::Runtime::CreationOptions::ExternalProfilingOptions options; options.m_EnableProfiling = true; @@ -2356,91 +2279,24 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadRequestCounterDirectoryPacket) // Bring the profiling service to the "Active" state BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - helper.ForceTransitionToState(ProfilingState::NotConnected); + profilingService.Update(); // Initialize the counter directory BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); profilingService.Update(); // Create the profiling connection BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); profilingService.Update(); // Start the command handler and the send thread - // Wait for the Stream Metadata packet the be sent - // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); - - // Force the profiling service to the "Active" state - helper.ForceTransitionToState(ProfilingState::Active); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far - mockProfilingConnection->Clear(); - - // Write a valid "Request Counter Directory" packet into the mock profiling connection, to simulate a valid - // reply from an external profiling service - - // Request Counter Directory packet header (word 0, word 1 is always zero): - // 26:31 [6] packet_family: Control Packet Family, value 0b000000 - // 16:25 [10] packet_id: Packet identifier, value 0b0000000011 - // 8:15 [8] reserved: Reserved, value 0b00000000 - // 0:7 [8] reserved: Reserved, value 0b00000000 - uint32_t packetFamily = 0; - uint32_t packetId = 123; // Wrong packet id!!! - uint32_t header = ((packetFamily & 0x0000003F) << 26) | ((packetId & 0x000003FF) << 16); - - // Create the Request Counter Directory packet - Packet requestCounterDirectoryPacket(header); - - // Write the packet to the mock profiling connection - mockProfilingConnection->WritePacket(std::move(requestCounterDirectoryPacket)); - - // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that - // the Create the Request Counter packet gets processed by the profiling service - std::this_thread::sleep_for(std::chrono::seconds(2)); - - // Check that the expected error has occurred and logged to the standard output - BOOST_CHECK(boost::contains(ss.str(), "Functor with requested PacketId=123 and Version=4194304 does not exist")); - - // The Request Counter Directory Command Handler should not have updated the profiling state - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - - // Reset the profiling service to stop any running thread - options.m_EnableProfiling = false; - profilingService.ResetExternalProfilingOptions(options, true); -} - -BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodRequestCounterDirectoryPacket) -{ - // Swap the profiling connection factory in the profiling service instance with our mock one - SwapProfilingConnectionFactoryHelper helper; - - // Reset the profiling service to the uninitialized state - armnn::Runtime::CreationOptions::ExternalProfilingOptions options; - options.m_EnableProfiling = true; - ProfilingService& profilingService = ProfilingService::Instance(); - profilingService.ResetExternalProfilingOptions(options, true); - - // Bring the profiling service to the "Active" state - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - profilingService.Update(); // Initialize the counter directory - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); - profilingService.Update(); // Create the profiling connection - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); - profilingService.Update(); // Start the command handler and the send thread - // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far mockProfilingConnection->Clear(); @@ -2463,7 +2319,7 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodRequestCounterDirectoryPacket) mockProfilingConnection->WritePacket(std::move(requestCounterDirectoryPacket)); // Wait for the Counter Directory packet to be sent - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection, 5000); // Check that the mock profiling connection contains one Counter Directory packet const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); @@ -2479,18 +2335,11 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodRequestCounterDirectoryPacket) profilingService.ResetExternalProfilingOptions(options, true); } -BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadPeriodicCounterSelectionPacket) +BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadPeriodicCounterSelectionPacketInvalidCounterUid) { - // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output - LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); - // Swap the profiling connection factory in the profiling service instance with our mock one SwapProfilingConnectionFactoryHelper helper; - // Redirect the standard output to a local stream so that we can parse the warning message - std::stringstream ss; - StreamRedirector streamRedirector(std::cout, ss.rdbuf()); - // Reset the profiling service to the uninitialized state armnn::Runtime::CreationOptions::ExternalProfilingOptions options; options.m_EnableProfiling = true; @@ -2505,88 +2354,18 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadPeriodicCounterSelectionPacket) BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); profilingService.Update(); // Start the command handler and the send thread - // Wait for the Stream Metadata packet the be sent - // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); - - // Force the profiling service to the "Active" state - helper.ForceTransitionToState(ProfilingState::Active); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far - mockProfilingConnection->Clear(); - - // Write a "Periodic Counter Selection" packet into the mock profiling connection, to simulate an input from an - // external profiling service - - // Periodic Counter Selection packet header: - // 26:31 [6] packet_family: Control Packet Family, value 0b000000 - // 16:25 [10] packet_id: Packet identifier, value 0b0000000100 - // 8:15 [8] reserved: Reserved, value 0b00000000 - // 0:7 [8] reserved: Reserved, value 0b00000000 - uint32_t packetFamily = 0; - uint32_t packetId = 999; // Wrong packet id!!! - uint32_t header = ((packetFamily & 0x0000003F) << 26) | ((packetId & 0x000003FF) << 16); - - // Create the Periodic Counter Selection packet - Packet periodicCounterSelectionPacket(header); // Length == 0, this will disable the collection of counters - - // Write the packet to the mock profiling connection - mockProfilingConnection->WritePacket(std::move(periodicCounterSelectionPacket)); - - // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that - // the Periodic Counter Selection packet gets processed by the profiling service - std::this_thread::sleep_for(std::chrono::seconds(2)); - - // Check that the expected error has occurred and logged to the standard output - BOOST_CHECK(boost::contains(ss.str(), "Functor with requested PacketId=999 and Version=4194304 does not exist")); - - // The Periodic Counter Selection Handler should not have updated the profiling state - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - - // Reset the profiling service to stop any running thread - options.m_EnableProfiling = false; - profilingService.ResetExternalProfilingOptions(options, true); -} - -BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadPeriodicCounterSelectionPacketInvalidCounterUid) -{ - // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output - LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); - - // Swap the profiling connection factory in the profiling service instance with our mock one - SwapProfilingConnectionFactoryHelper helper; - - // Reset the profiling service to the uninitialized state - armnn::Runtime::CreationOptions::ExternalProfilingOptions options; - options.m_EnableProfiling = true; - ProfilingService& profilingService = ProfilingService::Instance(); - profilingService.ResetExternalProfilingOptions(options, true); - - // Bring the profiling service to the "Active" state - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); - profilingService.Update(); // Initialize the counter directory - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); - profilingService.Update(); // Create the profiling connection - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); - profilingService.Update(); // Start the command handler and the send thread - // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far mockProfilingConnection->Clear(); @@ -2632,7 +2411,7 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadPeriodicCounterSelectionPacketInval // Keep waiting until all the expected packets have been received do { - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); if (writtenData.empty()) { @@ -2676,18 +2455,18 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketNoCo BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); profilingService.Update(); // Start the command handler and the send thread + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far mockProfilingConnection->Clear(); @@ -2710,7 +2489,7 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketNoCo mockProfilingConnection->WritePacket(std::move(periodicCounterSelectionPacket)); // Wait for the Periodic Counter Selection packet to be sent - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection, 5000); // The Periodic Counter Selection Handler should not have updated the profiling state BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); @@ -2744,18 +2523,18 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketSing BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); profilingService.Update(); // Start the command handler and the send thread + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far mockProfilingConnection->Clear(); @@ -2799,7 +2578,7 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketSing // Keep waiting until all the expected packets have been received do { - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); if (writtenData.empty()) { @@ -2828,7 +2607,6 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketMult { // Swap the profiling connection factory in the profiling service instance with our mock one SwapProfilingConnectionFactoryHelper helper; - // Reset the profiling service to the uninitialized state armnn::Runtime::CreationOptions::ExternalProfilingOptions options; options.m_EnableProfiling = true; @@ -2843,18 +2621,18 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketMult BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); profilingService.Update(); // Start the command handler and the send thread + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far mockProfilingConnection->Clear(); @@ -2900,7 +2678,7 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPeriodicCounterSelectionPacketMult // Keep waiting until all the expected packets have been received do { - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); if (writtenData.empty()) { @@ -2929,7 +2707,6 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceDisconnect) { // Swap the profiling connection factory in the profiling service instance with our mock one SwapProfilingConnectionFactoryHelper helper; - // Reset the profiling service to the uninitialized state armnn::Runtime::CreationOptions::ExternalProfilingOptions options; options.m_EnableProfiling = true; @@ -2956,23 +2733,23 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceDisconnect) // Try to disconnect the profiling service while in the "Active" state profilingService.Update(); // Start the command handler and the send thread + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Check that the profiling connection is open BOOST_CHECK(mockProfilingConnection->IsOpen()); profilingService.Disconnect(); - BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); // The state should have changed + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); // The state should have changed // Check that the profiling connection has been reset mockProfilingConnection = helper.GetMockProfilingConnection(); @@ -2987,7 +2764,6 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPerJobCounterSelectionPacket) { // Swap the profiling connection factory in the profiling service instance with our mock one SwapProfilingConnectionFactoryHelper helper; - // Reset the profiling service to the uninitialized state armnn::Runtime::CreationOptions::ExternalProfilingOptions options; options.m_EnableProfiling = true; @@ -3002,18 +2778,18 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPerJobCounterSelectionPacket) BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); profilingService.Update(); // Start the command handler and the send thread + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + // Wait for the Stream Metadata packet the be sent // (we are not testing the connection acknowledgement here so it will be ignored by this test) - helper.WaitForProfilingPacketsSent(); + helper.WaitForProfilingPacketsSent(mockProfilingConnection); // Force the profiling service to the "Active" state helper.ForceTransitionToState(ProfilingState::Active); BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); - // Get the mock profiling connection - MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); - BOOST_CHECK(mockProfilingConnection); - // Remove the packets received so far mockProfilingConnection->Clear(); @@ -3037,7 +2813,7 @@ BOOST_AUTO_TEST_CASE(CheckProfilingServiceGoodPerJobCounterSelectionPacket) // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that // the Per-Job Counter Selection packet gets processed by the profiling service - std::this_thread::sleep_for(std::chrono::seconds(2)); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); // The Per-Job Counter Selection packets are dropped silently, so there should be no reply coming // from the profiling service @@ -3079,4 +2855,323 @@ BOOST_AUTO_TEST_CASE(CheckConfigureProfilingServiceOff) profilingService.ResetExternalProfilingOptions(options, true); } +BOOST_AUTO_TEST_CASE(CheckProfilingServiceEnabled) +{ + // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output + LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); + armnn::Runtime::CreationOptions::ExternalProfilingOptions options; + options.m_EnableProfiling = true; + ProfilingService& profilingService = ProfilingService::Instance(); + profilingService.ResetExternalProfilingOptions(options, true); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + profilingService.Update(); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); + + // Redirect the output to a local stream so that we can parse the warning message + std::stringstream ss; + StreamRedirector streamRedirector(std::cout, ss.rdbuf()); + profilingService.Update(); + streamRedirector.CancelRedirect(); + + // Check that the expected error has occurred and logged to the standard output + if (!boost::contains(ss.str(), "Cannot connect to stream socket: Connection refused")) + { + std::cout << ss.str(); + BOOST_FAIL("Expected string not found."); + } + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); +} + +BOOST_AUTO_TEST_CASE(CheckProfilingServiceEnabledRuntime) +{ + // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output + LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); + armnn::Runtime::CreationOptions::ExternalProfilingOptions options; + ProfilingService& profilingService = ProfilingService::Instance(); + profilingService.ResetExternalProfilingOptions(options, true); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + profilingService.Update(); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + options.m_EnableProfiling = true; + profilingService.ResetExternalProfilingOptions(options); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + profilingService.Update(); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); + + // Redirect the output to a local stream so that we can parse the warning message + std::stringstream ss; + StreamRedirector streamRedirector(std::cout, ss.rdbuf()); + profilingService.Update(); + + streamRedirector.CancelRedirect(); + + // Check that the expected error has occurred and logged to the standard output + if (!boost::contains(ss.str(), "Cannot connect to stream socket: Connection refused")) + { + std::cout << ss.str(); + BOOST_FAIL("Expected string not found."); + } + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); +} + +BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadConnectionAcknowledgedPacket) +{ + // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output + LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); + // Swap the profiling connection factory in the profiling service instance with our mock one + SwapProfilingConnectionFactoryHelper helper; + + // Redirect the standard output to a local stream so that we can parse the warning message + std::stringstream ss; + StreamRedirector streamRedirector(std::cout, ss.rdbuf()); + + // Calculate the size of a Stream Metadata packet + std::string processName = GetProcessName().substr(0, 60); + unsigned int processNameSize = processName.empty() ? 0 : boost::numeric_cast<unsigned int>(processName.size()) + 1; + unsigned int streamMetadataPacketsize = 118 + processNameSize; + + // Reset the profiling service to the uninitialized state + armnn::Runtime::CreationOptions::ExternalProfilingOptions options; + options.m_EnableProfiling = true; + ProfilingService& profilingService = ProfilingService::Instance(); + profilingService.ResetExternalProfilingOptions(options, true); + + // Bring the profiling service to the "WaitingForAck" state + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + profilingService.Update(); // Initialize the counter directory + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); + profilingService.Update(); // Create the profiling connection + + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + + // Remove the packets received so far + mockProfilingConnection->Clear(); + + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); + profilingService.Update(); + + // Wait for the Stream Metadata packet to be sent + helper.WaitForProfilingPacketsSent(mockProfilingConnection); + + // Check that the mock profiling connection contains one Stream Metadata packet + const std::vector<uint32_t> writtenData = mockProfilingConnection->GetWrittenData(); + if (writtenData.size() > 1) + { + // If this thread has been blocked for some time a second or more Stream Metadata packet could have been sent. + // In these cases make sure all packet are of length streamMetadataPacketsize + for(uint32_t packetLength : writtenData) + { + BOOST_TEST(packetLength == streamMetadataPacketsize); + } + } + else + { + BOOST_TEST(writtenData.size() == 1); + BOOST_TEST(writtenData[0] == streamMetadataPacketsize); + } + + // Write a valid "Connection Acknowledged" packet into the mock profiling connection, to simulate a valid + // reply from an external profiling service + + // Connection Acknowledged Packet header (word 0, word 1 is always zero): + // 26:31 [6] packet_family: Control Packet Family, value 0b000000 + // 16:25 [10] packet_id: Packet identifier, value 0b0000000001 + // 8:15 [8] reserved: Reserved, value 0b00000000 + // 0:7 [8] reserved: Reserved, value 0b00000000 + uint32_t packetFamily = 0; + uint32_t packetId = 37; // Wrong packet id!!! + uint32_t header = ((packetFamily & 0x0000003F) << 26) | ((packetId & 0x000003FF) << 16); + + // Create the Connection Acknowledged Packet + Packet connectionAcknowledgedPacket(header); + + // Write the packet to the mock profiling connection + mockProfilingConnection->WritePacket(std::move(connectionAcknowledgedPacket)); + + // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that + // the Connection Acknowledged packet gets processed by the profiling service + std::this_thread::sleep_for(std::chrono::milliseconds(7)); + + streamRedirector.CancelRedirect(); + + // Check that the expected error has occurred and logged to the standard output + if (!boost::contains(ss.str(), "Functor with requested PacketId=37 and Version=4194304 does not exist")) + { + std::cout << ss.str(); + BOOST_FAIL("Expected string not found."); + } + + // The Connection Acknowledged Command Handler should not have updated the profiling state + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); + + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); +} + +BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadRequestCounterDirectoryPacket) +{ + // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output + LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); + // Swap the profiling connection factory in the profiling service instance with our mock one + SwapProfilingConnectionFactoryHelper helper; + + // Redirect the standard output to a local stream so that we can parse the warning message + std::stringstream ss; + StreamRedirector streamRedirector(std::cout, ss.rdbuf()); + + // Reset the profiling service to the uninitialized state + armnn::Runtime::CreationOptions::ExternalProfilingOptions options; + options.m_EnableProfiling = true; + ProfilingService& profilingService = ProfilingService::Instance(); + profilingService.ResetExternalProfilingOptions(options, true); + + // Bring the profiling service to the "Active" state + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + helper.ForceTransitionToState(ProfilingState::NotConnected); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); + profilingService.Update(); // Create the profiling connection + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); + profilingService.Update(); // Start the command handler and the send thread + + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + + // Wait for the Stream Metadata packet the be sent + // (we are not testing the connection acknowledgement here so it will be ignored by this test) + helper.WaitForProfilingPacketsSent(mockProfilingConnection); + + // Force the profiling service to the "Active" state + helper.ForceTransitionToState(ProfilingState::Active); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); + + // Remove the packets received so far + mockProfilingConnection->Clear(); + + // Write a valid "Request Counter Directory" packet into the mock profiling connection, to simulate a valid + // reply from an external profiling service + + // Request Counter Directory packet header (word 0, word 1 is always zero): + // 26:31 [6] packet_family: Control Packet Family, value 0b000000 + // 16:25 [10] packet_id: Packet identifier, value 0b0000000011 + // 8:15 [8] reserved: Reserved, value 0b00000000 + // 0:7 [8] reserved: Reserved, value 0b00000000 + uint32_t packetFamily = 0; + uint32_t packetId = 123; // Wrong packet id!!! + uint32_t header = ((packetFamily & 0x0000003F) << 26) | ((packetId & 0x000003FF) << 16); + + // Create the Request Counter Directory packet + Packet requestCounterDirectoryPacket(header); + + // Write the packet to the mock profiling connection + mockProfilingConnection->WritePacket(std::move(requestCounterDirectoryPacket)); + + // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that + // the Create the Request Counter packet gets processed by the profiling service + std::this_thread::sleep_for(std::chrono::milliseconds(7)); + + streamRedirector.CancelRedirect(); + + // Check that the expected error has occurred and logged to the standard output + if (!boost::contains(ss.str(), "Functor with requested PacketId=123 and Version=4194304 does not exist")) + { + std::cout << ss.str(); + BOOST_FAIL("Expected string not found."); + } + + // The Request Counter Directory Command Handler should not have updated the profiling state + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); + + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); +} + +BOOST_AUTO_TEST_CASE(CheckProfilingServiceBadPeriodicCounterSelectionPacket) +{ + // Locally reduce log level to "Warning", as this test needs to parse a warning message from the standard output + LogLevelSwapper logLevelSwapper(armnn::LogSeverity::Warning); + // Swap the profiling connection factory in the profiling service instance with our mock one + SwapProfilingConnectionFactoryHelper helper; + + // Redirect the standard output to a local stream so that we can parse the warning message + std::stringstream ss; + StreamRedirector streamRedirector(std::cout, ss.rdbuf()); + + // Reset the profiling service to the uninitialized state + armnn::Runtime::CreationOptions::ExternalProfilingOptions options; + options.m_EnableProfiling = true; + ProfilingService& profilingService = ProfilingService::Instance(); + profilingService.ResetExternalProfilingOptions(options, true); + + // Bring the profiling service to the "Active" state + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Uninitialised); + profilingService.Update(); // Initialize the counter directory + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::NotConnected); + profilingService.Update(); // Create the profiling connection + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::WaitingForAck); + profilingService.Update(); // Start the command handler and the send thread + + // Get the mock profiling connection + MockProfilingConnection* mockProfilingConnection = helper.GetMockProfilingConnection(); + BOOST_CHECK(mockProfilingConnection); + + // Wait for the Stream Metadata packet the be sent + // (we are not testing the connection acknowledgement here so it will be ignored by this test) + helper.WaitForProfilingPacketsSent(mockProfilingConnection); + + // Force the profiling service to the "Active" state + helper.ForceTransitionToState(ProfilingState::Active); + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); + + // Remove the packets received so far + mockProfilingConnection->Clear(); + + // Write a "Periodic Counter Selection" packet into the mock profiling connection, to simulate an input from an + // external profiling service + + // Periodic Counter Selection packet header: + // 26:31 [6] packet_family: Control Packet Family, value 0b000000 + // 16:25 [10] packet_id: Packet identifier, value 0b0000000100 + // 8:15 [8] reserved: Reserved, value 0b00000000 + // 0:7 [8] reserved: Reserved, value 0b00000000 + uint32_t packetFamily = 0; + uint32_t packetId = 999; // Wrong packet id!!! + uint32_t header = ((packetFamily & 0x0000003F) << 26) | ((packetId & 0x000003FF) << 16); + + // Create the Periodic Counter Selection packet + Packet periodicCounterSelectionPacket(header); // Length == 0, this will disable the collection of counters + + // Write the packet to the mock profiling connection + mockProfilingConnection->WritePacket(std::move(periodicCounterSelectionPacket)); + + // Wait for a bit (must at least be the delay value of the mock profiling connection) to make sure that + // the Periodic Counter Selection packet gets processed by the profiling service + std::this_thread::sleep_for(std::chrono::milliseconds(7)); + + // Check that the expected error has occurred and logged to the standard output + streamRedirector.CancelRedirect(); + + // Check that the expected error has occurred and logged to the standard output + if (!boost::contains(ss.str(), "Functor with requested PacketId=999 and Version=4194304 does not exist")) + { + std::cout << ss.str(); + BOOST_FAIL("Expected string not found."); + } + + // The Periodic Counter Selection Handler should not have updated the profiling state + BOOST_CHECK(profilingService.GetCurrentState() == ProfilingState::Active); + + // Reset the profiling service to stop any running thread + options.m_EnableProfiling = false; + profilingService.ResetExternalProfilingOptions(options, true); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/profiling/test/ProfilingTests.hpp b/src/profiling/test/ProfilingTests.hpp index 86b5c315e2..65c182b622 100644 --- a/src/profiling/test/ProfilingTests.hpp +++ b/src/profiling/test/ProfilingTests.hpp @@ -46,7 +46,18 @@ public: : m_Stream(stream) , m_BackupBuffer(m_Stream.rdbuf(newStreamBuffer)) {} - ~StreamRedirector() { m_Stream.rdbuf(m_BackupBuffer); } + + ~StreamRedirector() { CancelRedirect(); } + + void CancelRedirect() + { + // Only cancel the redirect once. + if (m_BackupBuffer != nullptr ) + { + m_Stream.rdbuf(m_BackupBuffer); + m_BackupBuffer = nullptr; + } + } private: std::ostream& m_Stream; @@ -67,11 +78,21 @@ public: Packet ReadPacket(uint32_t timeout) override { - std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); - - // Return connection acknowledged packet - return Packet(65536); + // First time we're called return a connection ack packet. After that always timeout. + if (m_FirstCall) + { + m_FirstCall = false; + // Return connection acknowledged packet + return Packet(65536); + } + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); + throw armnn::TimeoutException("Simulate a timeout error\n"); + } } + + bool m_FirstCall = true; }; class TestProfilingConnectionTimeoutError : public TestProfilingConnectionBase @@ -83,31 +104,46 @@ public: Packet ReadPacket(uint32_t timeout) override { - std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); - - if (m_ReadRequests < 3) + // Return connection acknowledged packet after three timeouts + if (m_ReadRequests % 3 == 0) { - m_ReadRequests++; + std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); + ++m_ReadRequests; throw armnn::TimeoutException("Simulate a timeout error\n"); } - // Return connection acknowledged packet after three timeouts return Packet(65536); } + int ReadCalledCount() + { + return m_ReadRequests.load(); + } + private: - int m_ReadRequests; + std::atomic<int> m_ReadRequests; }; class TestProfilingConnectionArmnnError : public TestProfilingConnectionBase { public: + TestProfilingConnectionArmnnError() + : m_ReadRequests(0) + {} + Packet ReadPacket(uint32_t timeout) override { - std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); - + ++m_ReadRequests; throw armnn::Exception("Simulate a non-timeout error"); } + + int ReadCalledCount() + { + return m_ReadRequests.load(); + } + +private: + std::atomic<int> m_ReadRequests; }; class TestFunctorA : public CommandHandlerFunctor @@ -172,9 +208,17 @@ public: TransitionToState(ProfilingService::Instance(), newState); } - void WaitForProfilingPacketsSent() + void WaitForProfilingPacketsSent(MockProfilingConnection* mockProfilingConnection, uint32_t timeout = 1000) { - return WaitForPacketSent(ProfilingService::Instance()); + if (!mockProfilingConnection->HasWrittenData()) + { + WaitForPacketSent(ProfilingService::Instance(), timeout); + // It's possible the wait has timed out. Check there is some data. + if (!mockProfilingConnection->HasWrittenData()) + { + throw RuntimeException("ProfilingTests::WaitForProfilingPacketsSent timeout waiting for packet."); + } + } } private: diff --git a/src/profiling/test/SendCounterPacketTests.cpp b/src/profiling/test/SendCounterPacketTests.cpp index b33b62f10b..740ea33c9a 100644 --- a/src/profiling/test/SendCounterPacketTests.cpp +++ b/src/profiling/test/SendCounterPacketTests.cpp @@ -24,6 +24,9 @@ using namespace armnn::profiling; namespace { +// A short delay to wait for the thread to process a packet. +uint16_t constexpr WAIT_UNTIL_READABLE_MS = 100; + void SetNotConnectedProfilingState(ProfilingStateMachine& profilingStateMachine) { ProfilingState currentState = profilingStateMachine.GetCurrentState(); @@ -1790,7 +1793,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest0) sendCounterPacket.Start(mockProfilingConnection); BOOST_CHECK(sendCounterPacket.IsRunning()); - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.Stop(); BOOST_CHECK(!sendCounterPacket.IsRunning()); @@ -1811,7 +1814,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest1) // Interleaving writes and reads to/from the buffer with pauses to test that the send thread actually waits for // something to become available for reading - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); CounterDirectory counterDirectory; sendCounterPacket.SendStreamMetaDataPacket(); @@ -1824,7 +1827,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest1) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendCounterDirectoryPacket(counterDirectory); @@ -1834,7 +1837,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest1) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendPeriodicCounterCapturePacket(123u, { @@ -1848,7 +1851,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest1) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendPeriodicCounterCapturePacket(44u, { @@ -1886,7 +1889,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest1) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendPeriodicCounterSelectionPacket(1000u, { 1345u, 254u, 4536u, 408u, 54u, 6323u, 428u, 1u, 6u }); @@ -1898,7 +1901,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest1) // To test an exact value of the "read size" in the mock buffer, wait two seconds to allow the send thread to // read all what's remaining in the buffer - std::this_thread::sleep_for(std::chrono::seconds(2)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.Stop(); @@ -1922,7 +1925,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest2) // Adding many spurious "ready to read" signals throughout the test to check that the send thread is // capable of handling unnecessary read requests - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SetReadyToRead(); @@ -1937,7 +1940,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest2) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendCounterDirectoryPacket(counterDirectory); @@ -1948,7 +1951,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest2) sendCounterPacket.SetReadyToRead(); sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendPeriodicCounterCapturePacket(123u, { @@ -1962,13 +1965,13 @@ BOOST_AUTO_TEST_CASE(SendThreadTest2) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SetReadyToRead(); sendCounterPacket.SetReadyToRead(); sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SetReadyToRead(); sendCounterPacket.SendPeriodicCounterCapturePacket(44u, @@ -2009,7 +2012,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest2) sendCounterPacket.SetReadyToRead(); sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.SendPeriodicCounterSelectionPacket(1000u, { 1345u, 254u, 4536u, 408u, 54u, 6323u, 428u, 1u, 6u }); @@ -2021,7 +2024,7 @@ BOOST_AUTO_TEST_CASE(SendThreadTest2) // To test an exact value of the "read size" in the mock buffer, wait two seconds to allow the send thread to // read all what's remaining in the buffer - std::this_thread::sleep_for(std::chrono::seconds(2)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); sendCounterPacket.Stop(); @@ -2148,7 +2151,7 @@ BOOST_AUTO_TEST_CASE(SendThreadBufferTest) // Interleaving writes and reads to/from the buffer with pauses to test that the send thread actually waits for // something to become available for reading - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); // SendStreamMetaDataPacket sendCounterPacket.SendStreamMetaDataPacket(); @@ -2173,7 +2176,7 @@ BOOST_AUTO_TEST_CASE(SendThreadBufferTest) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); // The buffer is read by the send thread so it should not be in the readable buffer. auto readBuffer = bufferManager.GetReadableBuffer(); @@ -2211,7 +2214,7 @@ BOOST_AUTO_TEST_CASE(SendThreadBufferTest) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); // The buffer is read by the send thread so it should not be in the readable buffer. readBuffer = bufferManager.GetReadableBuffer(); @@ -2252,7 +2255,7 @@ BOOST_AUTO_TEST_CASE(SendThreadBufferTest) sendCounterPacket.SetReadyToRead(); - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); // The buffer is read by the send thread so it should not be in the readable buffer. readBuffer = bufferManager.GetReadableBuffer(); @@ -2345,7 +2348,7 @@ BOOST_AUTO_TEST_CASE(SendThreadSendStreamMetadataPacket1) // The profiling state is set to "Uninitialized", so the send thread should throw an exception // Wait a bit to make sure that the send thread is properly started - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); BOOST_CHECK_THROW(sendCounterPacket.Stop(), armnn::RuntimeException); } @@ -2363,7 +2366,7 @@ BOOST_AUTO_TEST_CASE(SendThreadSendStreamMetadataPacket2) // The profiling state is set to "NotConnected", so the send thread should throw an exception // Wait a bit to make sure that the send thread is properly started - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); BOOST_CHECK_THROW(sendCounterPacket.Stop(), armnn::RuntimeException); } @@ -2386,7 +2389,7 @@ BOOST_AUTO_TEST_CASE(SendThreadSendStreamMetadataPacket3) // The profiling state is set to "WaitingForAck", so the send thread should send a Stream Metadata packet // Wait for a bit to make sure that we get the packet - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); BOOST_CHECK_NO_THROW(sendCounterPacket.Stop()); @@ -2414,7 +2417,7 @@ BOOST_AUTO_TEST_CASE(SendThreadSendStreamMetadataPacket4) // The profiling state is set to "WaitingForAck", so the send thread should send a Stream Metadata packet // Wait for a bit to make sure that we get the packet - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); // Check that the profiling state is still "WaitingForAck" BOOST_TEST((profilingStateMachine.GetCurrentState() == ProfilingState::WaitingForAck)); @@ -2430,7 +2433,7 @@ BOOST_AUTO_TEST_CASE(SendThreadSendStreamMetadataPacket4) sendCounterPacket.SetReadyToRead(); // Wait for a bit to make sure that we get the packet - std::this_thread::sleep_for(std::chrono::milliseconds(100)); + std::this_thread::sleep_for(std::chrono::milliseconds(WAIT_UNTIL_READABLE_MS)); // Check that the profiling state is still "WaitingForAck" BOOST_TEST((profilingStateMachine.GetCurrentState() == ProfilingState::WaitingForAck)); diff --git a/src/profiling/test/SendCounterPacketTests.hpp b/src/profiling/test/SendCounterPacketTests.hpp index cd41418e2c..4395d81d16 100644 --- a/src/profiling/test/SendCounterPacketTests.hpp +++ b/src/profiling/test/SendCounterPacketTests.hpp @@ -67,11 +67,9 @@ public: Packet ReadPacket(uint32_t timeout) override { - // Simulate a delay in the reading process - std::this_thread::sleep_for(std::chrono::milliseconds(timeout)); - + // Simulate a delay in the reading process. The default timeout is way too long. + std::this_thread::sleep_for(std::chrono::milliseconds(5)); std::lock_guard<std::mutex> lock(m_Mutex); - return std::move(m_Packet); } @@ -84,6 +82,12 @@ public: return writtenData; } + const bool HasWrittenData() + { + std::lock_guard<std::mutex> lock(m_Mutex); + return !m_WrittenData.empty(); + } + void Clear() { std::lock_guard<std::mutex> lock(m_Mutex); |