Host specific TLS contexts

TechnikEmpire · TechnikEmpire · commit 6cffd9617ed0 · 2017-09-26T17:35:07.000-04:00
Added host specific TLS contexts that are cached, so we not longer share a single global context. Fixed the vast majority of *real* issues reported by badssl.com. Removed old manual ECDH negotation key generation for server contexts and replaced with an openSSL option to auto handle this process. Added i-case string hash and equality functors based on boost for i-case string key hashmaps. Fixes TechnikEmpire#127 Fixes TechnikEmpire#113
diff --git a/ide/msvc/AssemblyInfo.cpp b/ide/msvc/AssemblyInfo.cpp
@@ -39,7 +39,7 @@ using namespace System::Security::Permissions;
 // You can specify all the value or you can default the Revision and Build Numbers
 // by using the '*' as shown below:
 
-[assembly:AssemblyVersionAttribute("2.0.1")];
+[assembly:AssemblyVersionAttribute("2.0.2")];
 
 [assembly:ComVisible(false)];
 
diff --git a/ide/msvc/HttpFe.Common/Properties/AssemblyInfo.cs b/ide/msvc/HttpFe.Common/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 //
 // You can specify all the values or you can default the Build and Revision Numbers by using the '*'
 // as shown below: [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("2.0.1.0")]
-[assembly: AssemblyFileVersion("2.0.1.0")]
+[assembly: AssemblyVersion("2.0.2.0")]
+[assembly: AssemblyFileVersion("2.0.2.0")]
diff --git a/ide/msvc/HttpFe.Managed/Properties/AssemblyInfo.cs b/ide/msvc/HttpFe.Managed/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 //
 // You can specify all the values or you can default the Build and Revision Numbers by using the '*'
 // as shown below: [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("2.0.1.0")]
-[assembly: AssemblyFileVersion("2.0.1.0")]
+[assembly: AssemblyVersion("2.0.2.0")]
+[assembly: AssemblyFileVersion("2.0.2.0")]
diff --git a/ide/msvc/Tests/Program.cs b/ide/msvc/Tests/Program.cs
@@ -8,19 +8,20 @@
 using HttpFe.Common;
 using HttpFe.Managed;
 using System;
+using System.Threading;
 
 namespace Tests
 {
     internal class Program
     {
-        private static volatile bool s_running = false;
+        private static ManualResetEvent s_rstEvent = new ManualResetEvent(false);
 
         private static void Main(string[] args)
         {
             Console.CancelKeyPress += (sender, eArgs) =>
             {
                 Console.WriteLine("Ctrl+C detected. Terminating.");
-                s_running = false;
+                s_rstEvent.Set();
             };
 
             try
@@ -51,11 +52,7 @@ private static void RunProgram()
                 engine.OnError += OnError;
 
                 engine.Start();
-                s_running = true;
-
-                while(s_running)
-                {
-                }                    
+                s_rstEvent.WaitOne();
 
                 engine.Stop();
             }
diff --git a/ide/msvc/Tests/Properties/AssemblyInfo.cs b/ide/msvc/Tests/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("2.0.1.0")]
-[assembly: AssemblyFileVersion("2.0.1.0")]
+[assembly: AssemblyVersion("2.0.2.0")]
+[assembly: AssemblyFileVersion("2.0.2.0")]
diff --git a/ide/msvc/Tests/Tests.csproj b/ide/msvc/Tests/Tests.csproj
@@ -59,6 +59,16 @@
     <None Include="App.config" />
     <None Include="app.manifest" />
   </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HttpFe.Common\HttpFe.Common.csproj">
+      <Project>{64938257-0baa-4e25-9f44-715ff8c9828c}</Project>
+      <Name>HttpFe.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HttpFe.Managed\HttpFe.Managed.csproj">
+      <Project>{0940a14e-98a1-44ed-b6f0-a9370610a761}</Project>
+      <Name>HttpFe.Managed</Name>
+    </ProjectReference>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>
     <PostBuildEvent>xcopy /D /Y /S /E "$(SolutionDir)..\..\build\HttpFe.Managed\$(Configuration)\*.*" "$(TargetDir)"</PostBuildEvent>
diff --git a/ide/msvc/libhttpfilteringengine.vcxproj b/ide/msvc/libhttpfilteringengine.vcxproj
@@ -559,6 +559,7 @@ xcopy /Y "$(ProjectDir)..\..\deps\windivert\msvc\x64\*.sys" "$(OutDir)"</Command
     <ClInclude Include="..\..\src\te\httpengine\network\SocketTypes.hpp" />
     <ClInclude Include="..\..\src\te\httpengine\util\cb\EngineCallbackTypes.h" />
     <ClInclude Include="..\..\src\te\httpengine\util\cb\EventReporter.hpp" />
+    <ClInclude Include="..\..\src\te\httpengine\util\hash\StringHashUtils.hpp" />
     <ClInclude Include="..\..\src\te\util\http\KnownHttpHeaders.hpp" />
     <ClInclude Include="..\..\src\te\util\string\StringRefUtil.hpp" />
     <ClInclude Include="resource.h" />
diff --git a/ide/msvc/libhttpfilteringengine.vcxproj.filters b/ide/msvc/libhttpfilteringengine.vcxproj.filters
@@ -127,6 +127,9 @@
     <Filter Include="Header Files\cpprestsdk">
       <UniqueIdentifier>{4f0dbcad-9dd2-4ec2-88d0-b9db517d6c3d}</UniqueIdentifier>
     </Filter>
+    <Filter Include="Header Files\te\httpengine\util\hash">
+      <UniqueIdentifier>{e858d8bc-1735-4271-b0f2-155dc53f2a87}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="resource.h">
@@ -189,6 +192,9 @@
     <ClInclude Include="..\..\contrib\cpprestsdk\src\http\client\x509_cert_utilities.h">
       <Filter>Header Files\cpprestsdk</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\te\httpengine\util\hash\StringHashUtils.hpp">
+      <Filter>Header Files\te\httpengine\util\hash</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="AssemblyInfo.cpp">
diff --git a/nuget/HttpFilteringEngine.nuspec b/nuget/HttpFilteringEngine.nuspec
@@ -2,10 +2,10 @@
 <package >
   <metadata>
     <id>HttpFilteringEngine</id>
-    <version>2.0.1</version>
+    <version>2.0.2</version>
     <title>HttpFilteringEngine</title>
     <authors>TechnikEmpire</authors>
-	<releaseNotes>Forced .NET framework version to 4.6. Fixed IPV6 filtering so it now functions.</releaseNotes>
+	<releaseNotes>We now create an isolated, unique TLS context for every unique host connected to.</releaseNotes>
     <owners>TechnikEmpire</owners>
 	<projectUrl>https://github.com/TechnikEmpire/HttpFilteringEngine</projectUrl>
 	<requireLicenseAcceptance>true</requireLicenseAcceptance>
diff --git a/src/te/httpengine/mitm/secure/BaseInMemoryCertificateStore.cpp b/src/te/httpengine/mitm/secure/BaseInMemoryCertificateStore.cpp
@@ -275,24 +275,7 @@ namespace te
 
 						SSL_CTX_set_options(ctx->native_handle(), SSL_OP_CIPHER_SERVER_PREFERENCE);
 
-						EC_KEY* tmpNegotiationEcKey;
-
-						if (nullptr == (tmpNegotiationEcKey = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1)))
-						{
-							EVP_PKEY_free(spoofedCertKeypair);
-							X509_free(spoofedCert);
-							throw std::runtime_error(u8"In BaseInMemoryCertificateStore::GetServerContext(std::string, X509*) - Failed to allocate server context temporary negotiation EC key.");
-						}
-
-						if (EC_KEY_generate_key(tmpNegotiationEcKey) != 1)
-						{
-							EC_KEY_free(tmpNegotiationEcKey);
-							EVP_PKEY_free(spoofedCertKeypair);
-							X509_free(spoofedCert);
-							throw std::runtime_error(u8"In BaseInMemoryCertificateStore::GetServerContext(std::string, X509*) - Failed to generate server context temporary negotiation EC key.");
-						}
-
-						SSL_CTX_set_tmp_ecdh(ctx->native_handle(), tmpNegotiationEcKey);
+						SSL_CTX_set_ecdh_auto(ctx->native_handle(), 1);
 
 						bool atLeastOneInsert = false;
 
@@ -318,8 +301,7 @@ namespace te
 						{
 							// In this case, either the user has made an error and is duplicating data, or perhaps
 							// something more dirty is going on, where we have spoofed a certificate that is lying
-							// about its SN and or SAN's.
-							EC_KEY_free(tmpNegotiationEcKey);
+							// about its SN and or SAN's.							
 							EVP_PKEY_free(spoofedCertKeypair);
 							X509_free(spoofedCert);
 							delete ctx;
diff --git a/src/te/httpengine/mitm/secure/TlsCapableHttpAcceptor.hpp b/src/te/httpengine/mitm/secure/TlsCapableHttpAcceptor.hpp
@@ -293,7 +293,7 @@ namespace te
 							boost::system::error_code loadRootsError;							
 
 							
-							m_clientContext.set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::context::verify_fail_if_no_peer_cert);							
+							m_clientContext.set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::context::verify_fail_if_no_peer_cert);
 							m_clientContext.load_verify_file(m_caBundleAbsolutePath, loadRootsError);
 							
 							if (loadRootsError)
diff --git a/src/te/httpengine/mitm/secure/TlsCapableHttpBridge.cpp b/src/te/httpengine/mitm/secure/TlsCapableHttpBridge.cpp
@@ -17,6 +17,12 @@ namespace te
 		{
 			namespace secure
 			{	
+				template <typename T>
+				std::unordered_map<std::string, std::unique_ptr<boost::asio::ssl::context>, util::hash::ICaseStringHash, util::hash::ICaseStringEquality>  TlsCapableHttpBridge<T>::s_clientContexts;
+
+				template <typename T>
+				std::atomic_flag TlsCapableHttpBridge<T>::s_clientContextLock = ATOMIC_FLAG_INIT;
+
 				TlsCapableHttpBridge<network::TcpSocket>::TlsCapableHttpBridge(
 					boost::asio::io_service* service,
 					BaseInMemoryCertificateStore* certStore,
@@ -101,6 +107,7 @@ namespace te
 					m_request.reset(new http::HttpRequest());
 					m_response.reset(new http::HttpResponse());
 
+					// Init TLS peek buffer.
 					m_tlsPeekBuffer.reset(new std::array<char, TlsPeekBufferSize>());	
 
 					// XXX TODO - This is ugly, our bad design is showing. See notes in the
@@ -174,6 +181,60 @@ namespace te
 					return m_downstreamSocket;
 				}
 
+				template<class BridgeSocketType>
+				const void TlsCapableHttpBridge<BridgeSocketType>::InitClientContext(TlsCapableHttpBridge<BridgeSocketType>* bridgeCtx, boost::asio::ssl::stream<boost::asio::ip::tcp::socket>& sslStream, const std::string& hostname)
+				{
+					while (s_clientContextLock.test_and_set(std::memory_order_acquire))
+					{
+						cpu_relax();
+					}
+
+					auto existingCtx = s_clientContexts.find(hostname);
+					if (existingCtx != s_clientContexts.end())
+					{
+						std::unique_ptr<boost::asio::ssl::context>& uPtr = existingCtx->second;
+						
+						SSL_set_SSL_CTX(sslStream.native_handle(), uPtr->native_handle());
+					}
+					else
+					{
+						std::unique_ptr<boost::asio::ssl::context> newContext;
+						newContext.reset(new boost::asio::ssl::context(sslStream.get_io_service(), boost::asio::ssl::context::sslv23_client));
+
+						newContext->set_verify_mode(boost::asio::ssl::context::verify_peer | boost::asio::ssl::context::verify_fail_if_no_peer_cert);
+
+						SSL_CTX_set_cipher_list(newContext->native_handle(), u8"HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4");
+
+						SSL_CTX_set_ecdh_auto(newContext->native_handle(), 1);
+
+						auto defaultContext = SSL_get_SSL_CTX(sslStream.native_handle());
+						
+						if (defaultContext != nullptr)
+						{
+							auto defaultCertStore = SSL_CTX_get_cert_store(defaultContext);
+
+							if (defaultCertStore != nullptr)
+							{
+								SSL_CTX_set_cert_store(newContext->native_handle(), defaultCertStore);
+							}
+							else
+							{
+								bridgeCtx->ReportError(u8"In TlsCapableHttpBridge<BridgeSocketType>::InitClientContext(boost::asio::ssl::stream<boost::asio::ip::tcp::socket>&, const std::string&) - Failed to get default certificate store.");
+							}
+						}
+						else
+						{
+							bridgeCtx->ReportError(u8"In TlsCapableHttpBridge<BridgeSocketType>::InitClientContext(boost::asio::ssl::stream<boost::asio::ip::tcp::socket>&, const std::string&) - Failed to get default client context.");
+						}
+
+						SSL_set_SSL_CTX(sslStream.native_handle(), newContext->native_handle());
+
+						s_clientContexts.emplace(hostname, std::move(newContext));
+					}
+
+					s_clientContextLock.clear(std::memory_order_release);
+				}
+
 				template<>
 				boost::asio::ip::tcp::socket& TlsCapableHttpBridge<network::TlsSocket>::DownstreamSocket()
 				{
@@ -407,6 +468,9 @@ namespace te
 
 					if (!error)
 					{
+						// Set up our host specific client context.
+						InitClientContext(this, m_upstreamSocket, m_upstreamHost);
+
 						SetStreamTimeout(boost::posix_time::minutes(5));
 
 						SSL_set_tlsext_host_name(m_upstreamSocket.native_handle(), m_upstreamHost.c_str());
diff --git a/src/te/httpengine/mitm/secure/TlsCapableHttpBridge.hpp b/src/te/httpengine/mitm/secure/TlsCapableHttpBridge.hpp
@@ -17,6 +17,8 @@
 #include "../http/HttpResponse.hpp"
 #include "../../util/cb/EventReporter.hpp"
 #include "../../../util/http/KnownHttpHeaders.hpp"
+#include "../../util/hash/StringHashUtils.hpp"
+
 #include <memory>
 #include <atomic>
 #include <type_traits>
@@ -25,13 +27,13 @@
 
 	#if BOOST_ARCH_X86_32
 		#ifdef _MSC_VER
-			#define cpu_relax() _mm_pause()
+			#define cpu_relax() YieldProcessor()
 		#else
 			#define cpu_relax() asm volatile("pause" ::: "memory")
 		#endif
 	#elif BOOST_ARCH_X86_64
 		#ifdef _MSC_VER
-			#define cpu_relax() _mm_pause()
+			#define cpu_relax() YieldProcessor()
 		#else
 			#define cpu_relax() asm volatile("pause" ::: "memory")
 		#endif
@@ -398,6 +400,29 @@ namespace te
 					/// </summary>
 					std::unique_ptr< std::array<char, TlsPeekBufferSize> > m_tlsPeekBuffer = nullptr;
 
+					/// <summary>
+					/// We keep host specific contexts here globally.
+					/// </summary>	
+					static std::unordered_map<std::string, std::unique_ptr<boost::asio::ssl::context>, util::hash::ICaseStringHash, util::hash::ICaseStringEquality> s_clientContexts;
+
+					/// <summary>
+					/// Part of spinlock for getting host specific client context.
+					/// </summary>
+					static std::atomic_flag s_clientContextLock;
+
+					/// <summary>
+					/// Called once we discover the SNI hostname for a TLS connection. This will
+					/// either retrieve an existing context, or create and retrieve a context,
+					/// tailored to the host in question.
+					/// </summary>
+					/// <param name="sslStream">
+					/// A reference to the SSL stream to configure the context of.
+					/// </param>
+					/// <param name="hostname">
+					/// The hostname.
+					/// </param>
+					static const void InitClientContext(TlsCapableHttpBridge<BridgeSocketType>* bridgeCtx, boost::asio::ssl::stream<boost::asio::ip::tcp::socket>& sslStream, const std::string& hostname);
+
 					/// <summary>
 					/// Tells the minimum length that a peek read must be in order to even reach
 					/// the extensions area of a potentially accurate TLS client hello.
diff --git a/src/te/httpengine/util/hash/StringHashUtils.hpp b/src/te/httpengine/util/hash/StringHashUtils.hpp
@@ -0,0 +1,66 @@
+/*
+* Copyright � 2017 Jesse Nicholson
+* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/.
+*/
+
+#pragma once
+
+#include <string>
+#include <cctype>
+#include <algorithm>
+#include <locale>
+#include <boost/algorithm/string.hpp>
+#include <boost/functional/hash.hpp>
+
+namespace te
+{
+	namespace httpengine
+	{
+		namespace util
+		{
+			namespace hash
+			{				
+				struct ICaseStringHash
+				{
+					size_t operator()(const std::string& str) const
+					{
+						std::size_t seed = 0;
+						std::locale locale;
+
+						for (std::string::const_iterator it = str.begin(); it != str.end(); ++it)
+						{
+							boost::hash_combine(seed, std::toupper(*it, locale));
+						}
+
+						return seed;
+					}
+				};
+
+				struct ICaseStringEquality
+				{
+					bool operator()(const std::string& str1, const std::string& str2) const
+					{
+						auto oneSize = str1.size();
+						auto twoSize = str2.size();
+
+						if (oneSize != twoSize)
+						{
+							return false;
+						}
+
+						std::locale locale;
+
+						if (std::toupper(str1[0], locale) != std::toupper(str2[0], locale))
+						{
+							return false;
+						}
+
+						return boost::algorithm::iequals(str1, str2, locale);
+					}
+				};
+			}
+		}
+	}
+}
