Skip to content

Commit cf114fc

Browse files
author
Sheng Yang
committed
Enable SSL for mgmt servers and agents
The port remains 8250. The keystore saved at /etc/cloud/management/cloud.keystore. We also include one fail-safe keystore/certificate for fallback if we are unable to generate certificate and keystore. If we use fail-safe keystore, a warning and calltrace would be show. Notice you need to upgrade agent, as well as systemVM's images.
1 parent 0256428 commit cf114fc

8 files changed

Lines changed: 484 additions & 47 deletions

File tree

‎agent/src/com/cloud/agent/Agent.java‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -366,28 +366,30 @@ protected void reconnect(final Link link) {
366366

367367
_resource.disconnected();
368368

369-
while (true) {
369+
int inProgress = 0;
370+
do {
370371
_shell.getBackoffAlgorithm().waitBeforeRetry();
371372

372373
s_logger.info("Lost connection to the server. Reconnecting....");
373374

374-
int inProgress = 0;
375-
if ((inProgress = _inProgress.get()) > 0) {
375+
inProgress = _inProgress.get();
376+
if (inProgress > 0) {
376377
s_logger.info("Cannot connect because we still have " + inProgress + " commands in progress.");
377-
continue;
378378
}
379+
} while (inProgress > 0);
379380

380-
try {
381-
final SocketChannel sch = SocketChannel.open();
382-
sch.configureBlocking(false);
383-
sch.connect(link.getSocketAddress());
384-
385-
link.connect(sch);
386-
return;
387-
} catch(final IOException e) {
388-
s_logger.error("Unable to establish connection with the server", e);
389-
}
390-
}
381+
_connection.stop();
382+
_connection = new NioClient(
383+
"Agent",
384+
_shell.getHost(),
385+
_shell.getPort(),
386+
_shell.getWorkers(),
387+
this);
388+
do {
389+
s_logger.info("Reconnecting...");
390+
_connection.start();
391+
_shell.getBackoffAlgorithm().waitBeforeRetry();
392+
} while (!_connection.isStartup());
391393
}
392394

393395
public void processStartupAnswer(Answer answer, Response response, Link link) {

‎build/build-cloud.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,13 @@
174174
<path refid="deps.classpath" />
175175
</path>
176176
<target name="compile-utils" depends="-init" description="Compile the utilities jar that is shared.">
177-
<compile-java jar.name="${utils.jar}" top.dir="${utils.dir}" classpath="utils.classpath" />
177+
<compile-java jar.name="${utils.jar}" top.dir="${utils.dir}" classpath="utils.classpath" >
178+
<include-files>
179+
<fileset dir="${utils.dir}/certs">
180+
<include name="*.keystore" />
181+
</fileset>
182+
</include-files>
183+
</compile-java>
178184
</target>
179185

180186
<property name="api.dir" location="${base.dir}/api" />

‎server/src/com/cloud/server/ConfigurationServerImpl.java‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,13 @@
2525
import java.io.FileNotFoundException;
2626
import java.io.FileOutputStream;
2727
import java.io.IOException;
28+
import java.io.OutputStreamWriter;
29+
import java.io.Writer;
2830
import java.math.BigInteger;
31+
import java.net.InetAddress;
2932
import java.net.NetworkInterface;
3033
import java.net.SocketException;
34+
import java.net.UnknownHostException;
3135
import java.security.MessageDigest;
3236
import java.security.NoSuchAlgorithmException;
3337
import java.sql.PreparedStatement;
@@ -36,6 +40,7 @@
3640
import java.util.List;
3741
import java.util.Properties;
3842
import java.util.UUID;
43+
import java.util.regex.Pattern;
3944

4045
import javax.crypto.KeyGenerator;
4146
import javax.crypto.SecretKey;
@@ -82,6 +87,7 @@
8287
import com.cloud.utils.component.ComponentLocator;
8388
import com.cloud.utils.db.DB;
8489
import com.cloud.utils.db.Transaction;
90+
import com.cloud.utils.encoding.Base64.OutputStream;
8591
import com.cloud.utils.exception.CloudRuntimeException;
8692
import com.cloud.utils.net.NetUtils;
8793
import com.cloud.utils.script.Script;
@@ -221,6 +227,9 @@ public void persistDefaultValues() throws InternalErrorException {
221227
}
222228
}
223229
}
230+
231+
// keystore for SSL/TLS connection
232+
updateSSLKeystore();
224233

225234
// store the public and private keys in the database
226235
updateKeyPairs();
@@ -369,6 +378,131 @@ protected void updateCloudIdentifier() {
369378
}
370379
}
371380

381+
private String getBase64Keystore(String keystorePath) throws IOException {
382+
byte[] storeBytes = new byte[4094];
383+
int len = 0;
384+
try {
385+
len = new FileInputStream(keystorePath).read(storeBytes);
386+
} catch (EOFException e) {
387+
} catch (Exception e) {
388+
throw new IOException("Cannot read the generated keystore file");
389+
}
390+
if (len > 3000) { // Base64 codec would enlarge data by 1/3, and we have 4094 bytes in database entry at most
391+
throw new IOException("KeyStore is too big for database! Length " + len);
392+
}
393+
394+
byte[] encodeBytes = new byte[len];
395+
System.arraycopy(storeBytes, 0, encodeBytes, 0, len);
396+
397+
return new String(Base64.encodeBase64(encodeBytes));
398+
}
399+
400+
@DB
401+
private void createSSLKeystoreDBEntry(String encodedKeystore) throws IOException {
402+
String insertSQL = "INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) " +
403+
"VALUES ('Hidden','DEFAULT', 'management-server','ssl.keystore', '" + encodedKeystore +"','SSL Keystore for the management servers')";
404+
Transaction txn = Transaction.currentTxn();
405+
try {
406+
PreparedStatement stmt = txn.prepareAutoCloseStatement(insertSQL);
407+
stmt.executeUpdate();
408+
if (s_logger.isDebugEnabled()) {
409+
s_logger.debug("SSL Keystore inserted into database");
410+
}
411+
} catch (SQLException ex) {
412+
s_logger.error("SQL of the SSL Keystore failed", ex);
413+
throw new IOException("SQL of the SSL Keystore failed");
414+
}
415+
}
416+
417+
private void generateDefaultKeystore(String keystorePath) throws IOException {
418+
String cn = "Cloudstack User";
419+
String ou;
420+
421+
try {
422+
ou = InetAddress.getLocalHost().getCanonicalHostName();
423+
String[] group = ou.split("\\.");
424+
425+
// Simple check to see if we got IP Address...
426+
boolean isIPAddress = Pattern.matches("[0-9]$", group[group.length - 1]);
427+
if (isIPAddress) {
428+
ou = "cloud.com";
429+
} else {
430+
ou = group[group.length - 1];
431+
for (int i = group.length - 2; i >= 0 && i >= group.length - 3; i--)
432+
ou = group[i] + "." + ou;
433+
}
434+
} catch (UnknownHostException ex) {
435+
s_logger.info("Fail to get user's domain name. Would use cloud.com. ", ex);
436+
ou = "cloud.com";
437+
}
438+
439+
String o = ou;
440+
String c = "Unknown";
441+
String dname = "cn=" + cn + ", ou=" + ou +", o=" + o + ", c=" + c;
442+
Script script = new Script(true, "keytool", 5000, null);
443+
script.add("-genkey");
444+
script.add("-keystore", keystorePath);
445+
script.add("-storepass", "vmops.com");
446+
script.add("-keypass", "vmops.com");
447+
script.add("-validity", "3650");
448+
script.add("-dname", dname);
449+
String result = script.execute();
450+
if (result != null) {
451+
throw new IOException("Fail to generate certificate!");
452+
}
453+
}
454+
455+
protected void updateSSLKeystore() {
456+
if (s_logger.isInfoEnabled()) {
457+
s_logger.info("Processing updateSSLKeyStore");
458+
}
459+
460+
String dbString = _configDao.getValue("ssl.keystore");
461+
String keystorePath = "/etc/cloud/management/cloud.keystore";
462+
File keystoreFile = new File(keystorePath);
463+
boolean dbExisted = (dbString != null && !dbString.isEmpty());
464+
465+
try {
466+
if (!dbExisted) {
467+
if (!keystoreFile.exists()) {
468+
generateDefaultKeystore(keystorePath);
469+
s_logger.info("Generated SSL keystore.");
470+
}
471+
String base64Keystore = getBase64Keystore(keystorePath);
472+
createSSLKeystoreDBEntry(base64Keystore);
473+
s_logger.info("Stored SSL keystore to database.");
474+
} else if (keystoreFile.exists()) { // and dbExisted
475+
// Check if they are the same one, otherwise override with local keystore
476+
String base64Keystore = getBase64Keystore(keystorePath);
477+
if (base64Keystore.compareTo(dbString) != 0) {
478+
_configDao.update("ssl.keystore", base64Keystore);
479+
s_logger.info("Updated database keystore with local one.");
480+
}
481+
} else { // !keystoreFile.exists() and dbExisted
482+
// Export keystore to local file
483+
byte[] storeBytes = Base64.decodeBase64(dbString);
484+
try {
485+
String tmpKeystorePath = "/tmp/tmpkey";
486+
FileOutputStream fo = new FileOutputStream(tmpKeystorePath);
487+
fo.write(storeBytes);
488+
fo.close();
489+
Script script = new Script(true, "cp", 5000, null);
490+
script.add(tmpKeystorePath);
491+
script.add(keystorePath);
492+
String result = script.execute();
493+
if (result != null) {
494+
throw new IOException();
495+
}
496+
} catch (Exception e) {
497+
throw new IOException("Fail to create keystore file!", e);
498+
}
499+
s_logger.info("Stored database keystore to local.");
500+
}
501+
} catch (Exception ex) {
502+
s_logger.warn("Would use fail-safe keystore to continue.", ex);
503+
}
504+
}
505+
372506
@DB
373507
protected void updateKeyPairs() {
374508
// Grab the SSH key pair and insert it into the database, if it is not present

‎utils/certs/cloud.keystore‎

1.18 KB
Binary file not shown.

‎utils/src/com/cloud/utils/nio/Link.java‎

Lines changed: 61 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@
2828
import java.nio.channels.WritableByteChannel;
2929
import java.util.concurrent.ConcurrentLinkedQueue;
3030

31+
import javax.net.ssl.SSLEngine;
32+
import javax.net.ssl.SSLEngineResult;
33+
import javax.net.ssl.SSLSession;
34+
import javax.net.ssl.SSLEngineResult.HandshakeStatus;
35+
3136
import org.apache.log4j.Logger;
3237

3338
/**
@@ -44,6 +49,8 @@ public class Link {
4449
private Object _attach;
4550
private boolean _readSize;
4651

52+
private SSLEngine _sslEngine;
53+
4754
public Link(InetSocketAddress addr, NioConnection connection) {
4855
_addr = addr;
4956
_connection = connection;
@@ -70,6 +77,10 @@ public void setKey(SelectionKey key) {
7077
_key = key;
7178
}
7279

80+
public void setSSLEngine(SSLEngine sslEngine) {
81+
_sslEngine = sslEngine;
82+
}
83+
7384
/**
7485
* Static methods for reading from a channel in case
7586
* you need to add a client that doesn't require nio.
@@ -190,8 +201,24 @@ public byte[] read(SocketChannel ch) throws IOException {
190201
}
191202

192203
_readBuffer.flip();
193-
byte[] result = new byte[_readBuffer.limit()];
194-
_readBuffer.get(result);
204+
205+
ByteBuffer appBuf;
206+
207+
SSLSession sslSession = _sslEngine.getSession();
208+
SSLEngineResult engResult;
209+
210+
//TODO may need to adjust the buffer size
211+
appBuf = ByteBuffer.allocate(sslSession.getApplicationBufferSize() + 40);
212+
engResult = _sslEngine.unwrap(_readBuffer, appBuf);
213+
if (engResult.getHandshakeStatus() != HandshakeStatus.FINISHED &&
214+
engResult.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING &&
215+
engResult.getStatus() != SSLEngineResult.Status.OK) {
216+
throw new IOException("SSL: SSLEngine return bad result! " + engResult);
217+
}
218+
219+
byte[] result = new byte[appBuf.position()];
220+
appBuf.flip();
221+
appBuf.get(result);
195222
_readBuffer.clear();
196223
_readSize = true;
197224

@@ -258,29 +285,46 @@ public boolean write(SocketChannel ch) throws IOException {
258285
return true;
259286
}
260287

261-
data[0].mark();
262-
int remaining = data[0].getInt() + 4;
263-
data[0].reset();
264-
265-
if (remaining > 65535) {
266-
throw new IOException("Fail to send a too big packet! Size: " + remaining);
288+
ByteBuffer pkgBuf;
289+
SSLSession sslSession = _sslEngine.getSession();
290+
SSLEngineResult engResult;
291+
292+
ByteBuffer headBuf = ByteBuffer.allocate(4);
293+
ByteBuffer[] raw_data = new ByteBuffer[data.length - 1];
294+
System.arraycopy(data, 1, raw_data, 0, data.length - 1);
295+
296+
pkgBuf = ByteBuffer.allocate(sslSession.getPacketBufferSize() + 40);
297+
engResult = _sslEngine.wrap(raw_data, pkgBuf);
298+
if (engResult.getHandshakeStatus() != HandshakeStatus.FINISHED &&
299+
engResult.getHandshakeStatus() != HandshakeStatus.NOT_HANDSHAKING &&
300+
engResult.getStatus() != SSLEngineResult.Status.OK) {
301+
throw new IOException("SSL: SSLEngine return bad result! " + engResult);
267302
}
268-
269-
while (remaining > 0) {
303+
304+
int dataRemaining = pkgBuf.position();
305+
int headRemaining = 4;
306+
pkgBuf.flip();
307+
headBuf.putInt(dataRemaining);
308+
headBuf.flip();
309+
310+
while (headRemaining > 0) {
270311
if (s_logger.isTraceEnabled()) {
271-
s_logger.trace("Writing " + remaining);
312+
s_logger.trace("Writing Header " + headRemaining);
272313
}
273-
long count = ch.write(data);
274-
remaining -= count;
314+
long count = ch.write(headBuf);
315+
headRemaining -= count;
316+
}
317+
while (dataRemaining > 0) {
318+
if (s_logger.isTraceEnabled()) {
319+
s_logger.trace("Writing Data " + dataRemaining);
320+
}
321+
long count = ch.write(pkgBuf);
322+
dataRemaining -= count;
275323
}
276324
}
277325
return false;
278326
}
279327

280-
public synchronized void connect(SocketChannel ch) {
281-
_connection.register(SelectionKey.OP_CONNECT, ch, this);
282-
}
283-
284328
public InetSocketAddress getSocketAddress() {
285329
return _addr;
286330
}

0 commit comments

Comments
 (0)