/* * Copyright (c) 2016 Jesse Nicholson. * * This file is part of Http Filtering Engine. * * Http Filtering Engine is free software: you can redistribute it and/or * modify it under the terms of the GNU General Public License as published * by the Free Software Foundation, either version 3 of the License, or (at * your option) any later version. * * In addition, as a special exception, the copyright holders give * permission to link the code of portions of this program with the OpenSSL * library. * * You must obey the GNU General Public License in all respects for all of * the code used other than OpenSSL. If you modify file(s) with this * exception, you may extend this exception to your version of the file(s), * but you are not obligated to do so. If you do not wish to do so, delete * this exception statement from your version. If you delete this exception * statement from all source files in the program, then also delete it * here. * * Http Filtering Engine is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General * Public License for more details. * * You should have received a copy of the GNU General Public License along * with Http Filtering Engine. If not, see . */ using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using BuildBot.Extensions; using BuildBot.Net.Http.Handlers; using BuildBotCore; using System.Security.Cryptography; using BuildBotCore.Common.ExternalTools.Compilers; using static BuildBotCore.Common.ExternalTools.Compilers.MSVCCompilerTask; using System.Threading; namespace HttpFilteringEngine { /// /// The BuildOpenSSL class handles compilation of OpenSSL for both x86 and /// x64 arch targets. This class will download the required tools, namely /// perl and nasm if such tools are not found in any existing environment /// variable. This class, or rather build task, will ensure that openSSL is /// configured, compiled and staged correctly provided that the user has /// Visual Studio 2015 installed with C/C++ support. All other requirements /// and configuration are handled herin. /// public class BuildOpenSSL : AbstractBuildTask { public override Guid GUID { get { return Guid.Parse("54917d60-831b-480b-b63e-e3a4f3c17994"); } } public override string Help { get { StringBuilder help = new StringBuilder(); help.AppendLine("In the event of an error, especially a \"previously failed configuration\" error, delete the openSSL submodule directory and re-initialize the submodule."); help.AppendLine("This can be done with \"git submodule update --init PATH\\TO\\OPENSSL\\SUBMODULE"); return help.ToString(); } } public override bool IsOsPlatformSupported { get { // XXX TODO - Update when other operating systems are supported. bool isWindows = System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows); return isWindows; } } public override Architecture SupportedArchitectures { get { return Architecture.x64 | Architecture.x86; } } public override List TaskDependencies { get { // Depends on the SetupSubmodules task. return new List(new[] {Guid.Parse("01241f94-9a80-42e9-bb23-f1470c40cff6")}); } } public override string TaskFriendlyName { get { return "OpenSSL Compilation"; } } public BuildOpenSSL(string scriptAbsolutePath) : base(scriptAbsolutePath) { } public override bool Clean() { // Clear errors before trying. Errors.Clear(); // XXX TODO - Need to build out the same paths as Run(), // get MSVC tools environment and then run nmake clean on // both the x86 and x64 dirs. Then, delete the output "MSVC" // folder. // Just return true for now. No harm. return true; } // Holds the path to the x86 source dir for openSSL. private string m_openSslx86Dir = string.Empty; // Holds the path to the x64 source dir for openSSL. private string m_openSslx64Dir = string.Empty; // Holds the path to the discovered or extracted NASM.exe. private string m_nasmDir = string.Empty; // Holds the path to the discovered or extracted PERL.exe. private string m_perlDir = string.Empty; public override bool Run(BuildConfiguration config, Architecture arch) { // Clear errors before trying. Errors.Clear(); if (!SupportedArchitectures.HasFlag(arch)) { Errors.Add(new Exception("Unsupported architecture specified for build task.")); return false; } if (!ConfigureDirectories()) { Errors.Add(new Exception("Failed to configure arch specific directories for openSSL build.")); return false; } // We need to get the environment for a the MSVC compiler and // associated build tools. var installedMsvcVersions = MSVCCompilerTask.InstalledToolVersions; if (installedMsvcVersions.Count == 0) { Errors.Add(new Exception("Could not detect a compatible installation of MSVC.")); return false; } // Get a reversed list of tool versions and iterate over them, until we find // an installed version. This way we're always working with the latest // version available. var allVersions = Enum.GetValues(typeof(ToolVersions)).Cast().Reverse(); ToolVersions versionToUse = ToolVersions.v11; foreach (var msvcVersion in allVersions) { if (installedMsvcVersions.ContainsKey(msvcVersion)) { versionToUse = msvcVersion; WriteLineToConsole(string.Format("Discovered and using MSVC {0} for compilation.", versionToUse.ToString())); break; } } // Build out the base path to the openSSL source directory. StringBuilder opensslBasePath = new StringBuilder(WorkingDirectory); opensslBasePath.Append(Path.DirectorySeparatorChar); opensslBasePath.Append("deps"); opensslBasePath.Append(Path.DirectorySeparatorChar); opensslBasePath.Append("openssl"); int numCompilationAttempts = 0; int numSuccessfulCompilations = 0; // We're only going to iterate over arches. We're not going to build a debug // version of openSSL, just release versions for each arch. foreach (Architecture a in Enum.GetValues(typeof(Architecture))) { if (arch.HasFlag(a)) { ++numCompilationAttempts; var finalBuildEnvironment = MSVCCompilerTask.GetEnvironmentForVersion(versionToUse, a); // Add perl path if it doesn't already exist. if (finalBuildEnvironment["PATH"].IndexOf(m_perlDir) == -1) { finalBuildEnvironment["PATH"] += (Path.PathSeparator + m_perlDir); } var configArgs = new List(); configArgs.Add("no-idea"); configArgs.Add("no-mdc2"); configArgs.Add("no-rc5"); configArgs.Add("no-comp"); // XXX TODO - Remove this option when upgrading to openSSL 1.1.0 configArgs.Add("no-ssl2"); configArgs.Add("no-ssl3"); configArgs.Add("no-weak-ssl-ciphers"); configArgs.Add("threads"); // The working dir. This will either be the x86 or x64 openSSL source dir. string workingDirectory = string.Empty; // We need to include nasm regardless of rater arch because // the openSSL configuration system will whine and quit if // we don't. We should be guaranteed to have a PATH variable // here unless something went horribly wrong. finalBuildEnvironment["PATH"] += (Path.PathSeparator + m_nasmDir); // XXX TODO - This needs to go away when we bump to OpenSSL 1.1.0 string whichAsmCall = string.Empty; string openSslInstallDir = string.Empty; switch (a) { case Architecture.x86: { // Build inside the x86 dir workingDirectory = m_openSslx86Dir; // Set x86 release build. configArgs.Insert(0, "VC-WIN32"); whichAsmCall = "ms" + Path.DirectorySeparatorChar + "do_nasm.bat"; openSslInstallDir = opensslBasePath.ToString().ConvertToHostOsPath() + Path.DirectorySeparatorChar + "msvc" + Path.DirectorySeparatorChar + "Releasex86"; } break; case Architecture.x64: { // Build inside the x64 dir workingDirectory = m_openSslx64Dir; whichAsmCall = "ms" + Path.DirectorySeparatorChar + "do_win64a.bat"; // Set x64 release build. configArgs.Insert(0, "VC-WIN64A"); openSslInstallDir = opensslBasePath.ToString().ConvertToHostOsPath() + Path.DirectorySeparatorChar + "msvc" + Path.DirectorySeparatorChar + "Releasex64"; } break; default: { WriteLineToConsole(string.Format("Dont have arch: {0}", a.ToString())); continue; } } // Setup prefix (output) path to deps/openssl/msvc/ReleaseX64 configArgs.Add( string.Format( "--prefix={0}", openSslInstallDir) ); // Setup config path to deps/openssl/msvc/ReleaseX86 configArgs.Add( string.Format( "--openssldir={0}", openSslInstallDir) ); WriteLineToConsole(string.Format("Configuring for arch: {0}", a.ToString())); WriteLineToConsole(workingDirectory); WriteLineToConsole(string.Format("Config Path: {0}", workingDirectory + Path.DirectorySeparatorChar + "Configure")); // Push configure script to front of args. configArgs.Insert(0, "Configure"); WriteLineToConsole(string.Join(" ", configArgs)); // Run the configuration process. var perlExitCode = RunProcess(workingDirectory, m_perlDir + Path.DirectorySeparatorChar + "perl.exe", configArgs, Timeout.Infinite, finalBuildEnvironment); // Now run the actual build process. // Example of the call string expanded/populated: // call "ms\do_nasm.bat" && nmake -f ms\ntdll.mak && nmake -f ms\ntdll.mak install string callArgs = string.Format("/C \"{0}\" && {1} && {2}", whichAsmCall, "nmake -f ms" + Path.DirectorySeparatorChar + "ntdll.mak", "nmake -f ms" + Path.DirectorySeparatorChar + "ntdll.mak install"); // XXX TODO - This is way to do it when we jump up to OpenSSL 1.1.0 //string callArgs = string.Format("/C {0} && {1}", "nmake", "nmake install"); // Running cmd.exe with these batch commands will build openSSL. var buildExitCode = RunProcess(workingDirectory, "cmd.exe", new List { callArgs }, Timeout.Infinite, finalBuildEnvironment); if(perlExitCode == 0 && buildExitCode == 0) { // Was a success. Move the output folder now. var destBaseDir = opensslBasePath.ToString().ConvertToHostOsPath() + Path.DirectorySeparatorChar + "msvc" + Path.DirectorySeparatorChar; var destReleaseDir = destBaseDir + string.Format("{0} {1}", BuildConfiguration.Release.ToString(), a.ToString()); var destDebugDir = destBaseDir + string.Format("{0} {1}", BuildConfiguration.Debug.ToString(), a.ToString()); // If we don't delete old stuff, Directory.Move will fail. if(Directory.Exists(destReleaseDir)) { Directory.Delete(destReleaseDir, true); } // Move aka rename the directory to have a space. try { Directory.Move(openSslInstallDir, destReleaseDir); } catch { // Sometimes getting access denied. Perhaps parts of the build // process are still hanging. Try and give them a few seconds // to wrap up, then try again. Thread.Sleep(3000); Directory.Move(openSslInstallDir, destReleaseDir); } // Simply copy the release folder for arch to a debug folder. CopyDirectory(destReleaseDir, destDebugDir, true); ++numSuccessfulCompilations; } } } var wasSuccess = numCompilationAttempts > 0 && numCompilationAttempts == numSuccessfulCompilations; return wasSuccess; } /// /// Ensures that arch-specific copies of the original source code are /// made and staged. /// /// /// True if the configuration was previously done or it was performed /// with success in this run. False otherwise. Failure is considered /// when an exception in this process has been raised and handled /// internally. /// private bool ConfigureDirectories() { try { // Build out the base path to the openSSL source directory. StringBuilder opensslBasePath = new StringBuilder(WorkingDirectory); opensslBasePath.Append(Path.DirectorySeparatorChar); opensslBasePath.Append("deps"); opensslBasePath.Append(Path.DirectorySeparatorChar); opensslBasePath.Append("openssl"); // Build out the x86 path. If this doesn't exist, then we have not yet // set up the two source copies. We need to move the source into a new // folder and clone it. The two folders will hold the openSSL source // configured for x86 and x64. Configuration modifies the source to the // point that separate compilation sources are necessary. m_openSslx86Dir = opensslBasePath.ToString() + Path.DirectorySeparatorChar + Architecture.x86.ToString(); m_openSslx64Dir = opensslBasePath.ToString() + Path.DirectorySeparatorChar + Architecture.x64.ToString(); if (!Directory.Exists(m_openSslx86Dir)) { // Start off by moving all the original source files to a new // directory titled "x86". Directory.CreateDirectory(m_openSslx86Dir); DirectoryInfo dirInfo = new DirectoryInfo(m_openSslx86Dir); List openSslSourceFiles = Directory.GetFiles(opensslBasePath.ToString(), "*.*", SearchOption.AllDirectories).ToList(); // Get the length of the base path. We'll cut this many // chars off to generate the new, moved base path. var basePathLength = opensslBasePath.ToString().Length; foreach (string file in openSslSourceFiles) { // Recreate the same path except based in our x86 directory. string newPath = dirInfo.FullName + Path.DirectorySeparatorChar + file.Substring(basePathLength); newPath = newPath.ConvertToHostOsPath(); // Ensure parent directory in new path exists. var parentDir = Directory.GetParent(newPath); if (!Directory.Exists(parentDir.FullName)) { parentDir.Create(); } FileInfo mFile = new FileInfo(file); if (new FileInfo(newPath).Exists == false) { mFile.MoveTo(newPath); } } // Now that the sources have been moved to "x86", we need to clone this into // a new folder called "x64". CopyDirectory(m_openSslx86Dir, m_openSslx64Dir, true); // Now delete all the empty directories that the file // moving left behind. var topLevelDirectories = Directory.GetDirectories(opensslBasePath.ToString(), "*.*", SearchOption.TopDirectoryOnly); foreach (var dir in topLevelDirectories) { if (Directory.GetFiles(dir, "*.*", SearchOption.AllDirectories).Length == 0) { Directory.Delete(dir, true); } } } else { // If this was done successfully before, then there ought to only be // two directories in our base openSSL directory. We specifically exclude directories // that have "msvc" in them, to not count the output directory for previously // successful builds. List openSslFolders = Directory.GetDirectories(opensslBasePath.ToString(), "*.*", SearchOption.TopDirectoryOnly).Where(file => !file.Contains("msvc")).ToList(); bool foundx86Dir = false; bool foundx64Dir = false; bool foundx86Files = false; bool foundx64Files = false; foreach (var listing in openSslFolders) { FileAttributes attr = File.GetAttributes(listing); if (attr.HasFlag(FileAttributes.Directory)) { var dirInfo = new DirectoryInfo(listing); if (dirInfo.Name.Equals(Architecture.x86.ToString(), StringComparison.OrdinalIgnoreCase)) { foundx86Dir = true; // Ensure that we have files in this directory. foundx86Files = Directory.GetFiles(dirInfo.FullName, "*.*", SearchOption.AllDirectories).Length > 0; } else if (dirInfo.Name.Equals(Architecture.x64.ToString(), StringComparison.OrdinalIgnoreCase)) { foundx64Dir = true; // Ensure that we have files in this directory. foundx64Files = Directory.GetFiles(dirInfo.FullName, "*.*", SearchOption.AllDirectories).Length > 0; } } } // Should have found both directories and they should have been the // only listings. bool previouslySucceeded = foundx86Dir && foundx64Dir && foundx86Files && foundx64Files && openSslFolders.Count == 2; if (!previouslySucceeded) { Errors.Add(new Exception("Possible previously failed or partial configuration detected.")); return false; } } } catch (Exception e) { // Something went wrong. Return false. Errors.Add(e); if (e.InnerException != null) { Errors.Add(e.InnerException); } return false; } // Configure perl. m_perlDir = ConfigurePerl(); if (string.IsNullOrEmpty(m_perlDir) || string.IsNullOrWhiteSpace(m_perlDir)) { // Failed to find perl. return false; } // Configure nasm. m_nasmDir = ConfigureNasm(); if (string.IsNullOrEmpty(m_nasmDir) || string.IsNullOrWhiteSpace(m_nasmDir)) { // Failed to find perl. return false; } WriteLineToConsole(m_perlDir); WriteLineToConsole(m_nasmDir); // As long as no exceptions were caught, we should be good. return true; } /// /// Ensures that we have perl available to us for the build process. If /// perl cannot be found in any environmental variable, then we'll fetch /// a portable copy. /// /// /// The full path to the parent directory of the perl primary executable. /// private string ConfigurePerl() { WriteLineToConsole("Searching for existing perl installations..."); var envVars = Environment.GetEnvironmentVariables(); foreach (var variable in envVars.Keys) { var split = ((string)envVars[variable]).Split(Path.PathSeparator); foreach (var val in split) { if (Directory.Exists(val)) { string perlPath = val.ConvertToHostOsPath() + Path.DirectorySeparatorChar + "perl.exe"; if (File.Exists(perlPath)) { return Directory.GetParent(perlPath).FullName.ConvertToHostOsPath(); } } } } // Means we didn't find perl. var toolsPath = WorkingDirectory + Path.DirectorySeparatorChar + "tools"; string strawberryPerlDownloadUri = string.Empty; string strawberryPerlSha1 = string.Empty; if (System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.HasFlag(System.Runtime.InteropServices.Architecture.X64)) { strawberryPerlDownloadUri = @"http://strawberryperl.com/download/5.24.0.1/strawberry-perl-5.24.0.1-64bit-portable.zip"; strawberryPerlSha1 = @"40094b93fdab1057598e9474767d34e810a1c383"; } else { strawberryPerlDownloadUri = @"http://strawberryperl.com/download/5.24.0.1/strawberry-perl-no64-5.24.0.1-32bit-portable.zip"; strawberryPerlSha1 = @"64fe479f4caa0881fca59e88c97d9cf2181a5007"; } var strawberryPerlZipName = "StrawberryPerl.zip"; var fullZipPath = toolsPath + Path.DirectorySeparatorChar + strawberryPerlZipName; bool zipAlreadyExists = File.Exists(fullZipPath); if (zipAlreadyExists) { WriteLineToConsole("Discovered previous download. Verifying integrity."); // Just let it revert to false if hash doesn't match. The file // would simply be overwritten. zipAlreadyExists = VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, strawberryPerlSha1); if (!zipAlreadyExists) { WriteLineToConsole("Integrity check failed. Attempting clean download."); } else { WriteLineToConsole("Integrity check passed. Using cached download."); } } if (!zipAlreadyExists) { var downloadTask = DownloadFile(strawberryPerlDownloadUri, toolsPath, null, strawberryPerlZipName); downloadTask.Wait(); if (!VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, strawberryPerlSha1)) { throw new Exception("Downloaded file does not match expected hash."); } } // Before decompressing again, let's see if we can find an already // decompressed perl.exe. var decompressedPath = toolsPath + Path.DirectorySeparatorChar + "strawberryperl"; string[] existingPerlPaths = new string[0]; if(Directory.Exists(decompressedPath)) { existingPerlPaths = Directory.GetFiles(decompressedPath, "perl.exe", SearchOption.AllDirectories); if (existingPerlPaths.Length > 0) { return Directory.GetParent(existingPerlPaths[0]).FullName.ConvertToHostOsPath(); } } // If we reached here, then we need to decompress. DecompressArchive(fullZipPath, decompressedPath); existingPerlPaths = Directory.GetFiles(toolsPath, "perl.exe", SearchOption.AllDirectories); if (existingPerlPaths.Length == 0) { WriteLineToConsole("Failed to find perl executable in extracted package."); return string.Empty; } return Directory.GetParent(existingPerlPaths[0]).FullName.ConvertToHostOsPath(); } private string ConfigureNasm() { WriteLineToConsole("Searching for existing nasm installations..."); var envVars = Environment.GetEnvironmentVariables(); foreach (var variable in envVars.Keys) { var split = ((string)envVars[variable]).Split(Path.PathSeparator); foreach (var val in split) { if (Directory.Exists(val)) { string nasmPath = val.ConvertToHostOsPath() + Path.DirectorySeparatorChar + "nasm.exe"; if (File.Exists(nasmPath)) { return Directory.GetParent(nasmPath).FullName.ConvertToHostOsPath(); } } } } // Means we didn't find nasm. var toolsPath = WorkingDirectory + Path.DirectorySeparatorChar + "tools"; string nasmDownloadUri = string.Empty; string nasmSha1 = string.Empty; if (System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.HasFlag(System.Runtime.InteropServices.Architecture.X64)) { nasmDownloadUri = @"http://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-win64.zip"; nasmSha1 = @"94756C0A427E65CD2AFE3DAC36F675BBAC3D89D8"; } else { nasmDownloadUri = @"http://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win32/nasm-2.12.02-win32.zip"; nasmSha1 = @"07D7C742DCC1107D7A322DB7A3A19065D7D1CBB4"; } var nasmZipName = "nasm.zip"; var fullZipPath = toolsPath + Path.DirectorySeparatorChar + nasmZipName; bool zipAlreadyExists = File.Exists(fullZipPath); if (zipAlreadyExists) { WriteLineToConsole("Discovered previous download. Verifying integrity."); // Just let it revert to false if hash doesn't match. The file // would simply be overwritten. zipAlreadyExists = VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, nasmSha1); if (!zipAlreadyExists) { WriteLineToConsole("Integrity check failed. Attempting clean download."); } else { WriteLineToConsole("Integrity check passed. Using cached download."); } } if (!zipAlreadyExists) { var downloadTask = DownloadFile(nasmDownloadUri, toolsPath, null, nasmZipName); downloadTask.Wait(); if (!VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, nasmSha1)) { throw new Exception("Downloaded file does not match expected hash."); } } // Before decompressing again, let's see if we can find an already // decompressed perl.exe. var decompressedPath = toolsPath + Path.DirectorySeparatorChar + "nasm"; string[] extractedNasmPaths = new string[0]; if(Directory.Exists(decompressedPath)) { extractedNasmPaths = Directory.GetFiles(decompressedPath, "nasm.exe", SearchOption.AllDirectories); if (extractedNasmPaths.Length > 0) { return Directory.GetParent(extractedNasmPaths[0]).FullName.ConvertToHostOsPath(); } } // If we reached here, then we need to decompress. DecompressArchive(fullZipPath, decompressedPath); extractedNasmPaths = Directory.GetFiles(toolsPath, "nasm.exe", SearchOption.AllDirectories); if (extractedNasmPaths.Length == 0) { WriteLineToConsole("Failed to find nasm executable in extracted package."); return string.Empty; } return Directory.GetParent(extractedNasmPaths[0]).FullName.ConvertToHostOsPath(); } } }