/*
* Copyright (c) 2016 Jesse Nicholson.
*
* This file is part of Http Filtering Engine.
*
* Http Filtering Engine is free software: you can redistribute it and/or
* modify it under the terms of the GNU General Public License as published
* by the Free Software Foundation, either version 3 of the License, or (at
* your option) any later version.
*
* In addition, as a special exception, the copyright holders give
* permission to link the code of portions of this program with the OpenSSL
* library.
*
* You must obey the GNU General Public License in all respects for all of
* the code used other than OpenSSL. If you modify file(s) with this
* exception, you may extend this exception to your version of the file(s),
* but you are not obligated to do so. If you do not wish to do so, delete
* this exception statement from your version. If you delete this exception
* statement from all source files in the program, then also delete it
* here.
*
* Http Filtering Engine is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
* Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with Http Filtering Engine. If not, see .
*/
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using BuildBot.Extensions;
using BuildBot.Net.Http.Handlers;
using BuildBotCore;
using System.Security.Cryptography;
using BuildBotCore.Common.ExternalTools.Compilers;
using static BuildBotCore.Common.ExternalTools.Compilers.MSVCCompilerTask;
using System.Threading;
namespace HttpFilteringEngine
{
///
/// The BuildOpenSSL class handles compilation of OpenSSL for both x86 and
/// x64 arch targets. This class will download the required tools, namely
/// perl and nasm if such tools are not found in any existing environment
/// variable. This class, or rather build task, will ensure that openSSL is
/// configured, compiled and staged correctly provided that the user has
/// Visual Studio 2015 installed with C/C++ support. All other requirements
/// and configuration are handled herin.
///
public class BuildOpenSSL : AbstractBuildTask
{
public override Guid GUID
{
get
{
return Guid.Parse("54917d60-831b-480b-b63e-e3a4f3c17994");
}
}
public override string Help
{
get
{
StringBuilder help = new StringBuilder();
help.AppendLine("In the event of an error, especially a \"previously failed configuration\" error, delete the openSSL submodule directory and re-initialize the submodule.");
help.AppendLine("This can be done with \"git submodule update --init PATH\\TO\\OPENSSL\\SUBMODULE");
return help.ToString();
}
}
public override bool IsOsPlatformSupported
{
get
{
// XXX TODO - Update when other operating systems are supported.
bool isWindows = System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows);
return isWindows;
}
}
public override Architecture SupportedArchitectures
{
get
{
return Architecture.x64 | Architecture.x86;
}
}
public override List TaskDependencies
{
get
{
// Depends on the SetupSubmodules task.
return new List(new[] {Guid.Parse("01241f94-9a80-42e9-bb23-f1470c40cff6")});
}
}
public override string TaskFriendlyName
{
get
{
return "OpenSSL Compilation";
}
}
public BuildOpenSSL(string scriptAbsolutePath) : base(scriptAbsolutePath)
{
}
public override bool Clean()
{
// Clear errors before trying.
Errors.Clear();
// XXX TODO - Need to build out the same paths as Run(),
// get MSVC tools environment and then run nmake clean on
// both the x86 and x64 dirs. Then, delete the output "MSVC"
// folder.
// Just return true for now. No harm.
return true;
}
// Holds the path to the x86 source dir for openSSL.
private string m_openSslx86Dir = string.Empty;
// Holds the path to the x64 source dir for openSSL.
private string m_openSslx64Dir = string.Empty;
// Holds the path to the discovered or extracted NASM.exe.
private string m_nasmDir = string.Empty;
// Holds the path to the discovered or extracted PERL.exe.
private string m_perlDir = string.Empty;
public override bool Run(BuildConfiguration config, Architecture arch)
{
// Clear errors before trying.
Errors.Clear();
if (!SupportedArchitectures.HasFlag(arch))
{
Errors.Add(new Exception("Unsupported architecture specified for build task."));
return false;
}
if (!ConfigureDirectories())
{
Errors.Add(new Exception("Failed to configure arch specific directories for openSSL build."));
return false;
}
// We need to get the environment for a the MSVC compiler and
// associated build tools.
var installedMsvcVersions = MSVCCompilerTask.InstalledToolVersions;
if (installedMsvcVersions.Count == 0)
{
Errors.Add(new Exception("Could not detect a compatible installation of MSVC."));
return false;
}
// Get a reversed list of tool versions and iterate over them, until we find
// an installed version. This way we're always working with the latest
// version available.
var allVersions = Enum.GetValues(typeof(ToolVersions)).Cast().Reverse();
ToolVersions versionToUse = ToolVersions.v11;
foreach (var msvcVersion in allVersions)
{
if (installedMsvcVersions.ContainsKey(msvcVersion))
{
versionToUse = msvcVersion;
WriteLineToConsole(string.Format("Discovered and using MSVC {0} for compilation.", versionToUse.ToString()));
break;
}
}
// Build out the base path to the openSSL source directory.
StringBuilder opensslBasePath = new StringBuilder(WorkingDirectory);
opensslBasePath.Append(Path.DirectorySeparatorChar);
opensslBasePath.Append("deps");
opensslBasePath.Append(Path.DirectorySeparatorChar);
opensslBasePath.Append("openssl");
int numCompilationAttempts = 0;
int numSuccessfulCompilations = 0;
// We're only going to iterate over arches. We're not going to build a debug
// version of openSSL, just release versions for each arch.
foreach (Architecture a in Enum.GetValues(typeof(Architecture)))
{
if (arch.HasFlag(a))
{
++numCompilationAttempts;
var finalBuildEnvironment = MSVCCompilerTask.GetEnvironmentForVersion(versionToUse, a);
// Add perl path if it doesn't already exist.
if (finalBuildEnvironment["PATH"].IndexOf(m_perlDir) == -1)
{
finalBuildEnvironment["PATH"] += (Path.PathSeparator + m_perlDir);
}
var configArgs = new List();
configArgs.Add("no-idea");
configArgs.Add("no-mdc2");
configArgs.Add("no-rc5");
configArgs.Add("no-comp");
// XXX TODO - Remove this option when upgrading to openSSL 1.1.0
configArgs.Add("no-ssl2");
configArgs.Add("no-ssl3");
configArgs.Add("no-weak-ssl-ciphers");
configArgs.Add("threads");
// The working dir. This will either be the x86 or x64 openSSL source dir.
string workingDirectory = string.Empty;
// We need to include nasm regardless of rater arch because
// the openSSL configuration system will whine and quit if
// we don't. We should be guaranteed to have a PATH variable
// here unless something went horribly wrong.
finalBuildEnvironment["PATH"] += (Path.PathSeparator + m_nasmDir);
// XXX TODO - This needs to go away when we bump to OpenSSL 1.1.0
string whichAsmCall = string.Empty;
string openSslInstallDir = string.Empty;
switch (a)
{
case Architecture.x86:
{
// Build inside the x86 dir
workingDirectory = m_openSslx86Dir;
// Set x86 release build.
configArgs.Insert(0, "VC-WIN32");
whichAsmCall = "ms" + Path.DirectorySeparatorChar + "do_nasm.bat";
openSslInstallDir = opensslBasePath.ToString().ConvertToHostOsPath() +
Path.DirectorySeparatorChar +
"msvc" +
Path.DirectorySeparatorChar +
"Releasex86";
}
break;
case Architecture.x64:
{
// Build inside the x64 dir
workingDirectory = m_openSslx64Dir;
whichAsmCall = "ms" + Path.DirectorySeparatorChar + "do_win64a.bat";
// Set x64 release build.
configArgs.Insert(0, "VC-WIN64A");
openSslInstallDir = opensslBasePath.ToString().ConvertToHostOsPath() +
Path.DirectorySeparatorChar +
"msvc" +
Path.DirectorySeparatorChar +
"Releasex64";
}
break;
default:
{
WriteLineToConsole(string.Format("Dont have arch: {0}", a.ToString()));
continue;
}
}
// Setup prefix (output) path to deps/openssl/msvc/ReleaseX64
configArgs.Add(
string.Format(
"--prefix={0}",
openSslInstallDir)
);
// Setup config path to deps/openssl/msvc/ReleaseX86
configArgs.Add(
string.Format(
"--openssldir={0}",
openSslInstallDir)
);
WriteLineToConsole(string.Format("Configuring for arch: {0}", a.ToString()));
WriteLineToConsole(workingDirectory);
WriteLineToConsole(string.Format("Config Path: {0}", workingDirectory + Path.DirectorySeparatorChar + "Configure"));
// Push configure script to front of args.
configArgs.Insert(0, "Configure");
WriteLineToConsole(string.Join(" ", configArgs));
// Run the configuration process.
var perlExitCode = RunProcess(workingDirectory, m_perlDir + Path.DirectorySeparatorChar + "perl.exe", configArgs, Timeout.Infinite, finalBuildEnvironment);
// Now run the actual build process.
// Example of the call string expanded/populated:
// call "ms\do_nasm.bat" && nmake -f ms\ntdll.mak && nmake -f ms\ntdll.mak install
string callArgs = string.Format("/C \"{0}\" && {1} && {2}", whichAsmCall, "nmake -f ms" + Path.DirectorySeparatorChar + "ntdll.mak", "nmake -f ms" + Path.DirectorySeparatorChar + "ntdll.mak install");
// XXX TODO - This is way to do it when we jump up to OpenSSL 1.1.0
//string callArgs = string.Format("/C {0} && {1}", "nmake", "nmake install");
// Running cmd.exe with these batch commands will build openSSL.
var buildExitCode = RunProcess(workingDirectory, "cmd.exe", new List { callArgs }, Timeout.Infinite, finalBuildEnvironment);
if(perlExitCode == 0 && buildExitCode == 0)
{
// Was a success. Move the output folder now.
var destBaseDir = opensslBasePath.ToString().ConvertToHostOsPath() +
Path.DirectorySeparatorChar +
"msvc" +
Path.DirectorySeparatorChar;
var destReleaseDir = destBaseDir + string.Format("{0} {1}", BuildConfiguration.Release.ToString(), a.ToString());
var destDebugDir = destBaseDir + string.Format("{0} {1}", BuildConfiguration.Debug.ToString(), a.ToString());
// If we don't delete old stuff, Directory.Move will fail.
if(Directory.Exists(destReleaseDir))
{
Directory.Delete(destReleaseDir, true);
}
// Move aka rename the directory to have a space.
try
{
Directory.Move(openSslInstallDir, destReleaseDir);
}
catch
{
// Sometimes getting access denied. Perhaps parts of the build
// process are still hanging. Try and give them a few seconds
// to wrap up, then try again.
Thread.Sleep(3000);
Directory.Move(openSslInstallDir, destReleaseDir);
}
// Simply copy the release folder for arch to a debug folder.
CopyDirectory(destReleaseDir, destDebugDir, true);
++numSuccessfulCompilations;
}
}
}
var wasSuccess = numCompilationAttempts > 0 && numCompilationAttempts == numSuccessfulCompilations;
return wasSuccess;
}
///
/// Ensures that arch-specific copies of the original source code are
/// made and staged.
///
///
/// True if the configuration was previously done or it was performed
/// with success in this run. False otherwise. Failure is considered
/// when an exception in this process has been raised and handled
/// internally.
///
private bool ConfigureDirectories()
{
try
{
// Build out the base path to the openSSL source directory.
StringBuilder opensslBasePath = new StringBuilder(WorkingDirectory);
opensslBasePath.Append(Path.DirectorySeparatorChar);
opensslBasePath.Append("deps");
opensslBasePath.Append(Path.DirectorySeparatorChar);
opensslBasePath.Append("openssl");
// Build out the x86 path. If this doesn't exist, then we have not yet
// set up the two source copies. We need to move the source into a new
// folder and clone it. The two folders will hold the openSSL source
// configured for x86 and x64. Configuration modifies the source to the
// point that separate compilation sources are necessary.
m_openSslx86Dir = opensslBasePath.ToString() + Path.DirectorySeparatorChar + Architecture.x86.ToString();
m_openSslx64Dir = opensslBasePath.ToString() + Path.DirectorySeparatorChar + Architecture.x64.ToString();
if (!Directory.Exists(m_openSslx86Dir))
{
// Start off by moving all the original source files to a new
// directory titled "x86".
Directory.CreateDirectory(m_openSslx86Dir);
DirectoryInfo dirInfo = new DirectoryInfo(m_openSslx86Dir);
List openSslSourceFiles = Directory.GetFiles(opensslBasePath.ToString(), "*.*", SearchOption.AllDirectories).ToList();
// Get the length of the base path. We'll cut this many
// chars off to generate the new, moved base path.
var basePathLength = opensslBasePath.ToString().Length;
foreach (string file in openSslSourceFiles)
{
// Recreate the same path except based in our x86 directory.
string newPath = dirInfo.FullName + Path.DirectorySeparatorChar + file.Substring(basePathLength);
newPath = newPath.ConvertToHostOsPath();
// Ensure parent directory in new path exists.
var parentDir = Directory.GetParent(newPath);
if (!Directory.Exists(parentDir.FullName))
{
parentDir.Create();
}
FileInfo mFile = new FileInfo(file);
if (new FileInfo(newPath).Exists == false)
{
mFile.MoveTo(newPath);
}
}
// Now that the sources have been moved to "x86", we need to clone this into
// a new folder called "x64".
CopyDirectory(m_openSslx86Dir, m_openSslx64Dir, true);
// Now delete all the empty directories that the file
// moving left behind.
var topLevelDirectories = Directory.GetDirectories(opensslBasePath.ToString(), "*.*", SearchOption.TopDirectoryOnly);
foreach (var dir in topLevelDirectories)
{
if (Directory.GetFiles(dir, "*.*", SearchOption.AllDirectories).Length == 0)
{
Directory.Delete(dir, true);
}
}
}
else
{
// If this was done successfully before, then there ought to only be
// two directories in our base openSSL directory. We specifically exclude directories
// that have "msvc" in them, to not count the output directory for previously
// successful builds.
List openSslFolders = Directory.GetDirectories(opensslBasePath.ToString(), "*.*", SearchOption.TopDirectoryOnly).Where(file => !file.Contains("msvc")).ToList();
bool foundx86Dir = false;
bool foundx64Dir = false;
bool foundx86Files = false;
bool foundx64Files = false;
foreach (var listing in openSslFolders)
{
FileAttributes attr = File.GetAttributes(listing);
if (attr.HasFlag(FileAttributes.Directory))
{
var dirInfo = new DirectoryInfo(listing);
if (dirInfo.Name.Equals(Architecture.x86.ToString(), StringComparison.OrdinalIgnoreCase))
{
foundx86Dir = true;
// Ensure that we have files in this directory.
foundx86Files = Directory.GetFiles(dirInfo.FullName, "*.*", SearchOption.AllDirectories).Length > 0;
}
else if (dirInfo.Name.Equals(Architecture.x64.ToString(), StringComparison.OrdinalIgnoreCase))
{
foundx64Dir = true;
// Ensure that we have files in this directory.
foundx64Files = Directory.GetFiles(dirInfo.FullName, "*.*", SearchOption.AllDirectories).Length > 0;
}
}
}
// Should have found both directories and they should have been the
// only listings.
bool previouslySucceeded = foundx86Dir && foundx64Dir && foundx86Files && foundx64Files && openSslFolders.Count == 2;
if (!previouslySucceeded)
{
Errors.Add(new Exception("Possible previously failed or partial configuration detected."));
return false;
}
}
}
catch (Exception e)
{
// Something went wrong. Return false.
Errors.Add(e);
if (e.InnerException != null)
{
Errors.Add(e.InnerException);
}
return false;
}
// Configure perl.
m_perlDir = ConfigurePerl();
if (string.IsNullOrEmpty(m_perlDir) || string.IsNullOrWhiteSpace(m_perlDir))
{
// Failed to find perl.
return false;
}
// Configure nasm.
m_nasmDir = ConfigureNasm();
if (string.IsNullOrEmpty(m_nasmDir) || string.IsNullOrWhiteSpace(m_nasmDir))
{
// Failed to find perl.
return false;
}
WriteLineToConsole(m_perlDir);
WriteLineToConsole(m_nasmDir);
// As long as no exceptions were caught, we should be good.
return true;
}
///
/// Ensures that we have perl available to us for the build process. If
/// perl cannot be found in any environmental variable, then we'll fetch
/// a portable copy.
///
///
/// The full path to the parent directory of the perl primary executable.
///
private string ConfigurePerl()
{
WriteLineToConsole("Searching for existing perl installations...");
var envVars = Environment.GetEnvironmentVariables();
foreach (var variable in envVars.Keys)
{
var split = ((string)envVars[variable]).Split(Path.PathSeparator);
foreach (var val in split)
{
if (Directory.Exists(val))
{
string perlPath = val.ConvertToHostOsPath() + Path.DirectorySeparatorChar + "perl.exe";
if (File.Exists(perlPath))
{
return Directory.GetParent(perlPath).FullName.ConvertToHostOsPath();
}
}
}
}
// Means we didn't find perl.
var toolsPath = WorkingDirectory + Path.DirectorySeparatorChar + "tools";
string strawberryPerlDownloadUri = string.Empty;
string strawberryPerlSha1 = string.Empty;
if (System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.HasFlag(System.Runtime.InteropServices.Architecture.X64))
{
strawberryPerlDownloadUri = @"http://strawberryperl.com/download/5.24.0.1/strawberry-perl-5.24.0.1-64bit-portable.zip";
strawberryPerlSha1 = @"40094b93fdab1057598e9474767d34e810a1c383";
}
else
{
strawberryPerlDownloadUri = @"http://strawberryperl.com/download/5.24.0.1/strawberry-perl-no64-5.24.0.1-32bit-portable.zip";
strawberryPerlSha1 = @"64fe479f4caa0881fca59e88c97d9cf2181a5007";
}
var strawberryPerlZipName = "StrawberryPerl.zip";
var fullZipPath = toolsPath + Path.DirectorySeparatorChar + strawberryPerlZipName;
bool zipAlreadyExists = File.Exists(fullZipPath);
if (zipAlreadyExists)
{
WriteLineToConsole("Discovered previous download. Verifying integrity.");
// Just let it revert to false if hash doesn't match. The file
// would simply be overwritten.
zipAlreadyExists = VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, strawberryPerlSha1);
if (!zipAlreadyExists)
{
WriteLineToConsole("Integrity check failed. Attempting clean download.");
}
else
{
WriteLineToConsole("Integrity check passed. Using cached download.");
}
}
if (!zipAlreadyExists)
{
var downloadTask = DownloadFile(strawberryPerlDownloadUri, toolsPath, null, strawberryPerlZipName);
downloadTask.Wait();
if (!VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, strawberryPerlSha1))
{
throw new Exception("Downloaded file does not match expected hash.");
}
}
// Before decompressing again, let's see if we can find an already
// decompressed perl.exe.
var decompressedPath = toolsPath + Path.DirectorySeparatorChar + "strawberryperl";
string[] existingPerlPaths = new string[0];
if(Directory.Exists(decompressedPath))
{
existingPerlPaths = Directory.GetFiles(decompressedPath, "perl.exe", SearchOption.AllDirectories);
if (existingPerlPaths.Length > 0)
{
return Directory.GetParent(existingPerlPaths[0]).FullName.ConvertToHostOsPath();
}
}
// If we reached here, then we need to decompress.
DecompressArchive(fullZipPath, decompressedPath);
existingPerlPaths = Directory.GetFiles(toolsPath, "perl.exe", SearchOption.AllDirectories);
if (existingPerlPaths.Length == 0)
{
WriteLineToConsole("Failed to find perl executable in extracted package.");
return string.Empty;
}
return Directory.GetParent(existingPerlPaths[0]).FullName.ConvertToHostOsPath();
}
private string ConfigureNasm()
{
WriteLineToConsole("Searching for existing nasm installations...");
var envVars = Environment.GetEnvironmentVariables();
foreach (var variable in envVars.Keys)
{
var split = ((string)envVars[variable]).Split(Path.PathSeparator);
foreach (var val in split)
{
if (Directory.Exists(val))
{
string nasmPath = val.ConvertToHostOsPath() + Path.DirectorySeparatorChar + "nasm.exe";
if (File.Exists(nasmPath))
{
return Directory.GetParent(nasmPath).FullName.ConvertToHostOsPath();
}
}
}
}
// Means we didn't find nasm.
var toolsPath = WorkingDirectory + Path.DirectorySeparatorChar + "tools";
string nasmDownloadUri = string.Empty;
string nasmSha1 = string.Empty;
if (System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.HasFlag(System.Runtime.InteropServices.Architecture.X64))
{
nasmDownloadUri = @"http://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win64/nasm-2.12.02-win64.zip";
nasmSha1 = @"94756C0A427E65CD2AFE3DAC36F675BBAC3D89D8";
}
else
{
nasmDownloadUri = @"http://www.nasm.us/pub/nasm/releasebuilds/2.12.02/win32/nasm-2.12.02-win32.zip";
nasmSha1 = @"07D7C742DCC1107D7A322DB7A3A19065D7D1CBB4";
}
var nasmZipName = "nasm.zip";
var fullZipPath = toolsPath + Path.DirectorySeparatorChar + nasmZipName;
bool zipAlreadyExists = File.Exists(fullZipPath);
if (zipAlreadyExists)
{
WriteLineToConsole("Discovered previous download. Verifying integrity.");
// Just let it revert to false if hash doesn't match. The file
// would simply be overwritten.
zipAlreadyExists = VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, nasmSha1);
if (!zipAlreadyExists)
{
WriteLineToConsole("Integrity check failed. Attempting clean download.");
}
else
{
WriteLineToConsole("Integrity check passed. Using cached download.");
}
}
if (!zipAlreadyExists)
{
var downloadTask = DownloadFile(nasmDownloadUri, toolsPath, null, nasmZipName);
downloadTask.Wait();
if (!VerifyFileHash(HashAlgorithmName.SHA1, fullZipPath, nasmSha1))
{
throw new Exception("Downloaded file does not match expected hash.");
}
}
// Before decompressing again, let's see if we can find an already
// decompressed perl.exe.
var decompressedPath = toolsPath + Path.DirectorySeparatorChar + "nasm";
string[] extractedNasmPaths = new string[0];
if(Directory.Exists(decompressedPath))
{
extractedNasmPaths = Directory.GetFiles(decompressedPath, "nasm.exe", SearchOption.AllDirectories);
if (extractedNasmPaths.Length > 0)
{
return Directory.GetParent(extractedNasmPaths[0]).FullName.ConvertToHostOsPath();
}
}
// If we reached here, then we need to decompress.
DecompressArchive(fullZipPath, decompressedPath);
extractedNasmPaths = Directory.GetFiles(toolsPath, "nasm.exe", SearchOption.AllDirectories);
if (extractedNasmPaths.Length == 0)
{
WriteLineToConsole("Failed to find nasm executable in extracted package.");
return string.Empty;
}
return Directory.GetParent(extractedNasmPaths[0]).FullName.ConvertToHostOsPath();
}
}
}