diff --git a/.gitignore b/.gitignore index 1c438f403..8864ca3e1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ target work *.iml +/target +.* diff --git a/pom.xml b/pom.xml index ec10d89fa..e0a3066ff 100644 --- a/pom.xml +++ b/pom.xml @@ -3,12 +3,12 @@ org.jenkins-ci.plugins plugin - 1.546 + 1.553 com.nirima docker-plugin - 0.2-SNAPSHOT + 0.2-rs-SNAPSHOT hpi Docker plugin @@ -23,14 +23,14 @@ - com.kpelykh + com.github.docker-java docker-java - 0.6.1-nm-SNAPSHOT + 0.9.1 org.jenkins-ci.main jenkins-core - 1.546 + 1.553 org.kohsuke.stapler diff --git a/src/main/java/com/nirima/jenkins/plugins/docker/DockerCloud.java b/src/main/java/com/nirima/jenkins/plugins/docker/DockerCloud.java index b75e29294..f733e43a2 100644 --- a/src/main/java/com/nirima/jenkins/plugins/docker/DockerCloud.java +++ b/src/main/java/com/nirima/jenkins/plugins/docker/DockerCloud.java @@ -3,24 +3,30 @@ import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.Collections2; -import com.kpelykh.docker.client.DockerClient; -import com.kpelykh.docker.client.DockerException; -import com.kpelykh.docker.client.model.Container; +import com.github.dockerjava.client.DockerClient; +import com.github.dockerjava.client.DockerException; +import com.github.dockerjava.client.model.Container; + import hudson.Extension; import hudson.model.*; +import hudson.model.MultiStageTimeSeries.TimeScale; import hudson.slaves.Cloud; import hudson.slaves.NodeProvisioner; +import hudson.slaves.NodeProvisioner.PlannedNode; import hudson.util.FormValidation; import hudson.util.StreamTaskListener; import jenkins.model.Jenkins; + import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.QueryParameter; -import org.kohsuke.stapler.StaplerResponse; import javax.annotation.Nullable; import javax.servlet.ServletException; + import java.io.IOException; +import java.lang.reflect.Field; import java.net.URL; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -53,8 +59,6 @@ public DockerCloud(String name, List templates, String else this.templates = Collections.emptyList(); - - readResolve(); } @@ -68,88 +72,88 @@ protected Object readResolve() { * Connects to Docker. */ public synchronized DockerClient connect() { - if (connection == null) { - connection = new DockerClient(serverUrl); + try { + connection = new DockerClient(serverUrl); + } + catch (DockerException e) { + LOGGER.log(Level.SEVERE, "Docker client creation failed " + e, e); + } } return connection; } @Override - public Collection provision(Label label, int excessWorkload) { + public Collection provision(final Label label, final int excessWorkload) { try { - // Count number of pending executors from spot requests - /* for(EC2SpotSlave n : NodeIterator.nodes(EC2SpotSlave.class)){ - // If the slave is online then it is already counted by Jenkins - // We only want to count potential additional Spot instance slaves - if (n.getComputer().isOffline()){ - DescribeSpotInstanceRequestsRequest dsir = - new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(n.getSpotInstanceRequestId()); - - for(SpotInstanceRequest sir : connect().describeSpotInstanceRequests(dsir).getSpotInstanceRequests()) { - // Count Spot requests that are open and still have a chance to be active - // A request can be active and not yet registered as a slave. We check above - // to ensure only unregistered slaves get counted - if(sir.getState().equals("open") || sir.getState().equals("active")){ - excessWorkload -= n.getNumExecutors(); - } - } - } - } - */ - LOGGER.log(Level.INFO, "Excess workload after pending Spot instances: " + excessWorkload); + LOGGER.log(Level.INFO, "Excess workload: " + excessWorkload); List r = new ArrayList(); final DockerTemplate t = getTemplate(label); - - while (excessWorkload>0) { - - if (!addProvisionedSlave(t.image, t.instanceCap)) { - break; - } - - r.add(new NodeProvisioner.PlannedNode(t.getDisplayName(), - Computer.threadPoolForRemoting.submit(new Callable() { - public Node call() throws Exception { - // TODO: record the output somewhere - try { - DockerSlave s = t.provision(new StreamTaskListener(System.out)); - Jenkins.getInstance().addNode(s); - // EC2 instances may have a long init script. If we declare - // the provisioning complete by returning without the connect - // operation, NodeProvisioner may decide that it still wants - // one more instance, because it sees that (1) all the slaves - // are offline (because it's still being launched) and - // (2) there's no capacity provisioned yet. - // - // deferring the completion of provisioning until the launch - // goes successful prevents this problem. - s.toComputer().connect(false).get(); - return s; - } - catch(Exception ex) { - LOGGER.log(Level.WARNING, "Error in provisioning"); - ex.printStackTrace(); - throw Throwables.propagate(ex); - } - finally { - //decrementAmiSlaveProvision(t.ami); - } - } - }) - ,t.getNumExecutors())); - - excessWorkload -= t.getNumExecutors(); - + int containersToCreate = Math.min(excessWorkload + t.minIdleContainers, t.instanceCap); + LOGGER.log(Level.INFO, "Creating " + containersToCreate + " containers..."); + + while (containersToCreate > 0) { + boolean provisioned = provisionContainer(t, r); + if (!provisioned) { + break; + } + containersToCreate -= t.getNumExecutors(); } return r; - } catch (Exception e) { - LOGGER.log(Level.WARNING,"Failed to count the # of live instances on EC2",e); + } + catch (Exception e) { + LOGGER.log(Level.WARNING,"Failed to provision Docker slave", e); return Collections.emptyList(); } } + + /** + * Provision a container slave. + * @param t + * @param r + * @return true if the slave could be provisioned, false if it could not and no other can be at this time. + */ + private boolean provisionContainer(final DockerTemplate t, List r) + throws Exception { + if (!addProvisionedSlave(t.image, t.instanceCap)) { + return false; + } + + r.add(new NodeProvisioner.PlannedNode(t.getDisplayName(), + Computer.threadPoolForRemoting.submit(new Callable() { + public Node call() throws Exception { + // TODO: record the output somewhere + try { + DockerSlave s = t.provision(new StreamTaskListener(System.out, Charset.defaultCharset())); + Jenkins.getInstance().addNode(s); + // EC2 instances may have a long init script. If we declare + // the provisioning complete by returning without the connect + // operation, NodeProvisioner may decide that it still wants + // one more instance, because it sees that (1) all the slaves + // are offline (because it's still being launched) and + // (2) there's no capacity provisioned yet. + // + // deferring the completion of provisioning until the launch + // goes successful prevents this problem. + s.toComputer().connect(false).get(); + return s; + } + catch(Exception ex) { + LOGGER.log(Level.WARNING, "Error in provisioning"); + ex.printStackTrace(); + throw Throwables.propagate(ex); + } + finally { + //decrementAmiSlaveProvision(t.ami); + } + } + }) + , t.getNumExecutors())); + return true; + } @Override public boolean canProvision(Label label) { @@ -181,20 +185,19 @@ public DockerTemplate getTemplate(Label label) { * Check not too many already running. * */ - private boolean addProvisionedSlave(String image, int amiCap) throws Exception { - if( amiCap == 0 ) + private boolean addProvisionedSlave(final String image, int instanceCap) throws Exception { + if( instanceCap == 0 ) return true; - List containers = connect().listContainers(false); + List containers = connect().listContainersCmd().withShowAll(false).exec(); Collection matching = Collections2.filter(containers, new Predicate() { public boolean apply(@Nullable Container container) { - // TODO: filter out containers not of the right type. - return true; + return image.equalsIgnoreCase(container.getImage()); } }); - return matching.size() < amiCap; + return matching.size() < instanceCap; } @Extension @@ -209,9 +212,40 @@ public FormValidation doTestConnection( ) throws IOException, ServletException, DockerException { DockerClient dc = new DockerClient(serverUrl.toString()); - dc.info(); + dc.infoCmd().exec(); return FormValidation.ok(); } } + + void containerTerminated(DockerTemplate template, DockerSlave dockerSlave, TaskListener listener) { + if (template.getMinIdleContainers() >= 0) { + // if we have a number of min idle containers, create them + Label label = Label.get(dockerSlave.getLabelString()); + LoadStatistics stat = label.loadStatistics; + NodeProvisioner provisioner = label.nodeProvisioner; + synchronized (provisioner) { + try { + // There may be a rounding error here. Jenkins is doing some odd stuff. + int excessWorkload = (int) Math.min(stat.queueLength.getLatest(TimeScale.SEC10), stat.computeQueueLength()); + excessWorkload -= label.getIdleExecutors(); + // This turned into a total hack. + Field pendingLaunchesField = provisioner.getClass().getDeclaredField("pendingLaunches"); + pendingLaunchesField.setAccessible(true); + List plannedNodes = (List) pendingLaunchesField.get(provisioner); + for (PlannedNode plannedNode : plannedNodes) { + excessWorkload -= plannedNode.numExecutors; + } + if (excessWorkload > -template.getMinIdleContainers()) { + plannedNodes.addAll(provision(label, excessWorkload)); + } + } + catch (Exception e) { + LOGGER.log(Level.WARNING, + "Error in provisioning Docker container to maintain minimum idle count: " + e.toString()); + } + } + } + + } } \ No newline at end of file diff --git a/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputer.java b/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputer.java index 905555d5e..bcbb3e9b7 100644 --- a/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputer.java +++ b/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputer.java @@ -2,9 +2,11 @@ import com.google.common.base.Objects; import com.nirima.jenkins.plugins.docker.action.DockerBuildAction; + import hudson.model.*; import hudson.slaves.AbstractCloudComputer; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; @@ -15,7 +17,7 @@ public class DockerComputer extends AbstractCloudComputer { private static final Logger LOGGER = Logger.getLogger(DockerComputer.class.getName()); - private boolean haveWeRunAnyJobs = false; + private AtomicBoolean haveWeRunAnyJobs = new AtomicBoolean(false); public DockerComputer(DockerSlave dockerSlave) { @@ -27,45 +29,52 @@ public DockerCloud getCloud() { } public boolean haveWeRunAnyJobs() { - return haveWeRunAnyJobs; + return haveWeRunAnyJobs.get(); } @Override public void taskAccepted(Executor executor, Queue.Task task) { super.taskAccepted(executor, task); - LOGGER.warning(" Computer " + this + " taskAccepted"); + LOGGER.log(Level.FINE, " Computer " + this + " taskAccepted"); } @Override public void taskCompleted(Executor executor, Queue.Task task, long durationMS) { - super.taskCompleted(executor, task, durationMS); - - haveWeRunAnyJobs = true; - - Queue.Executable executable = executor.getCurrentExecutable(); - if( executable instanceof Run) { - Run build = (Run) executable; - - if( getNode().dockerTemplate.tagOnCompletion ) { - getNode().commitOnTerminate( build ); - } - - - } - LOGGER.log(Level.INFO, " Computer " + this + " taskCompleted"); - + try { + LOGGER.log(Level.FINE, " Computer " + this + " taskCompleted"); + super.taskCompleted(executor, task, durationMS); + + Queue.Executable executable = executor.getCurrentExecutable(); + if( executable instanceof Run) { + Run build = (Run) executable; + + if( getNode().dockerTemplate.tagOnCompletion ) { + getNode().commitOnTerminate( build ); + } + } + } + finally { + haveWeRunAnyJobs.set(true); + getNode().retentionTerminate(); // terminate immediately, retention takes too long + } } @Override public void taskCompletedWithProblems(Executor executor, Queue.Task task, long durationMS, Throwable problems) { - super.taskCompletedWithProblems(executor, task, durationMS, problems); - LOGGER.log(Level.INFO, " Computer " + this + " taskCompletedWithProblems"); + try { + LOGGER.log(Level.FINE, " Computer " + this + " taskCompletedWithProblems"); + super.taskCompletedWithProblems(executor, task, durationMS, problems); + } + finally { + haveWeRunAnyJobs.set(true); + getNode().retentionTerminate(); // terminate immediately, retention takes too long + } } @Override public boolean isAcceptingTasks() { - boolean result = !haveWeRunAnyJobs && super.isAcceptingTasks(); - LOGGER.log(Level.INFO, " Computer " + this + " isAcceptingTasks " + result); + boolean result = !haveWeRunAnyJobs() && super.isAcceptingTasks(); + LOGGER.log(Level.FINE, " Computer " + this + " isAcceptingTasks " + result); return result; } @@ -79,6 +88,7 @@ public void onConnected(){ @Override public String toString() { return Objects.toStringHelper(this) + .add("name", super.getName()) .add("slave", getNode()) .toString(); } diff --git a/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputerLauncher.java b/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputerLauncher.java index 3144059a0..4c2ff709d 100644 --- a/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputerLauncher.java +++ b/src/main/java/com/nirima/jenkins/plugins/docker/DockerComputerLauncher.java @@ -1,38 +1,31 @@ package com.nirima.jenkins.plugins.docker; -import com.kpelykh.docker.client.model.ContainerInspectResponse; import hudson.Extension; -import hudson.model.*; +import hudson.model.TaskListener; +import hudson.model.Descriptor; import hudson.plugins.sshslaves.SSHConnector; import hudson.plugins.sshslaves.SSHLauncher; import hudson.slaves.ComputerLauncher; import hudson.slaves.SlaveComputer; import java.io.IOException; -import java.io.PrintStream; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import java.util.List; - - +import java.util.Map; +import java.util.Map.Entry; +import java.util.logging.Level; +import java.util.logging.Logger; -import hudson.model.TaskListener; -import hudson.slaves.ComputerLauncher; -import hudson.slaves.SlaveComputer; import jenkins.model.Jenkins; -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +import com.github.dockerjava.client.model.ContainerInspectResponse; +import com.github.dockerjava.client.model.ExposedPort; +import com.github.dockerjava.client.model.Ports.Binding; /** - * {@link hudson.slaves.ComputerLauncher} for EC2 that waits for the instance to really come up before proceeding to + * {@link hudson.slaves.ComputerLauncher} for Docker that waits for the instance to really come up before proceeding to * the real user-specified {@link hudson.slaves.ComputerLauncher}. * * @author Kohsuke Kawaguchi @@ -53,13 +46,29 @@ public DockerComputerLauncher(DockerTemplate template, ContainerInspectResponse @Override public void launch(SlaveComputer _computer, TaskListener listener) throws IOException, InterruptedException { SSHLauncher launcher = getSSHLauncher(); - launcher.launch(_computer, listener); - if( launcher.getConnection() == null ) { - LOGGER.log(Level.WARNING, "Couldn't launch Docker template. Closing."); + int attemptsRemaining = 4; + while (launcher.getConnection() == null && attemptsRemaining > 0) { + synchronized (launcher) { + Thread.sleep(1000L * (5 - attemptsRemaining)); // sleep for a few seconds - 1, 2, 3, 4 + } + launcher.launch(_computer, listener); + if (launcher.getConnection() == null) { + attemptsRemaining--; + String message = "Failed to ssh to Docker container to install agent."; + if (attemptsRemaining > 0) { + message += " Retrying ssh agent installation " + attemptsRemaining + " more times."; + } + LOGGER.log(Level.WARNING, message); + } + } + if (launcher.getConnection() == null ) { + LOGGER.log(Level.WARNING, "Could not ssh install agent to Docker container. Closing container."); DockerComputer dc = (DockerComputer)_computer; dc.getNode().terminate(); } - LOGGER.log(Level.INFO, "Launched " + _computer); + else { + LOGGER.log(Level.INFO, "Launched " + _computer); + } } public SSHLauncher getSSHLauncher() throws MalformedURLException { @@ -68,16 +77,32 @@ public SSHLauncher getSSHLauncher() throws MalformedURLException { * id='970d68eb7410bca37ccc8ac193ae68a324f7d286012c1994dcf58a28daa76da2', created='2014-01-09T12:19:37.322591068Z', * path='/usr/sbin/sshd', args=[-D], * config=ContainerConfig{hostName=970d68eb7410, portSpecs=null, user=, tty=false, stdinOpen=false, stdInOnce=false, memoryLimit=0, memorySwap=0, cpuShares=0, attachStdin=false, attachStdout=false, attachStderr=false, env=null, cmd=[Ljava.lang.String;@658782a7, dns=null, image=jenkins-3, volumes=null, volumesFrom=, entrypoint=null, networkDisabled=false, privileged=false, workingDir=, domainName=, exposedPorts={22/tcp={}}}, state=ContainerState{running=true, pid=8032, exitCode=0, startedAt='2014-01-09T12:19:37.400471534Z', ghost=false, finishedAt='0001-01-01T00:00:00Z'}, image='0ca6c5d5135db3ffb8abfef6a0861a0d2e44b6f37a33b4012a3f2d5cc99f68e9', - * networkSettings=NetworkSettings{ipAddress='172.17.0.58', ipPrefixLen=16, gateway='172.17.42.1', bridge='docker0', ports={22/tcp=[Lcom.kpelykh.docker.client.model.PortBinding;@2392d604}}, sysInitPath='null', resolvConfPath='/etc/resolv.conf', volumes={}, volumesRW={}, hostnamePath='/var/lib/docker/containers/970d68eb7410bca37ccc8ac193ae68a324f7d286012c1994dcf58a28daa76da2/hostname', hostsPath='/var/lib/docker/containers/970d68eb7410bca37ccc8ac193ae68a324f7d286012c1994dcf58a28daa76da2/hosts', name='/prickly_turing', driver='aufs'} - + * networkSettings=NetworkSettings{ipAddress='172.17.0.58', ipPrefixLen=16, gateway='172.17.42.1', bridge='docker0', ports={22/tcp=[Lcom.github.dockerjava.client.model.PortBinding;@2392d604}}, sysInitPath='null', resolvConfPath='/etc/resolv.conf', volumes={}, volumesRW={}, hostnamePath='/var/lib/docker/containers/970d68eb7410bca37ccc8ac193ae68a324f7d286012c1994dcf58a28daa76da2/hostname', hostsPath='/var/lib/docker/containers/970d68eb7410bca37ccc8ac193ae68a324f7d286012c1994dcf58a28daa76da2/hosts', name='/prickly_turing', driver='aufs'} */ - int port = Integer.parseInt(detail.getNetworkSettings().ports.get("22/tcp")[0].hostPort); + int hostPort = -1; + Map portBindingMap = detail.getNetworkSettings().getPorts().getBindings(); + for (Entry portBinding : portBindingMap.entrySet()) { + if (22 == portBinding.getKey().getPort()) { + hostPort = Integer.valueOf(portBinding.getValue().getHostPort()); + break; + } + } + if (hostPort == -1) { + throw new RuntimeException("Host port not found for the SSH port"); + } URL hostUrl = new URL(template.getParent().serverUrl); + String host = hostUrl.getHost(); + + LOGGER.log(Level.INFO, "Creating slave SSH launcher for " + host + ":" + hostPort); - LOGGER.log(Level.INFO, "Attempting launch on" + hostUrl + " port " + port); - - return new SSHLauncher(hostUrl.getHost(), port, template.credentialsId, template.jvmOptions , template.javaPath, template.prefixStartSlaveCmd, template.suffixStartSlaveCmd); + return new SSHLauncher(host, hostPort, template.credentialsId, template.jvmOptions , template.javaPath, template.prefixStartSlaveCmd, template.suffixStartSlaveCmd); + } + + @Override + public void beforeDisconnect(SlaveComputer computer, TaskListener listener) { + LOGGER.log(Level.INFO, "Disconnecting " + computer); + super.beforeDisconnect(computer, listener); } @Extension diff --git a/src/main/java/com/nirima/jenkins/plugins/docker/DockerRetentionStrategy.java b/src/main/java/com/nirima/jenkins/plugins/docker/DockerRetentionStrategy.java index 722386c41..41bff5fe3 100644 --- a/src/main/java/com/nirima/jenkins/plugins/docker/DockerRetentionStrategy.java +++ b/src/main/java/com/nirima/jenkins/plugins/docker/DockerRetentionStrategy.java @@ -2,13 +2,12 @@ import hudson.model.Descriptor; import hudson.slaves.RetentionStrategy; -import hudson.util.TimeUnit2; -import org.kohsuke.stapler.DataBoundConstructor; -import java.io.IOException; import java.util.logging.Level; import java.util.logging.Logger; +import org.kohsuke.stapler.DataBoundConstructor; + public class DockerRetentionStrategy extends RetentionStrategy { @@ -21,20 +20,14 @@ public DockerRetentionStrategy() { @Override public synchronized long check(DockerComputer c) { - LOGGER.log(Level.INFO, "Checking " + c); + LOGGER.log(Level.FINE, "Checking " + c); if (c.isIdle() && c.isOnline() && !disabled && c.haveWeRunAnyJobs()) { // TODO: really think about the right strategy here final long idleMilliseconds = System.currentTimeMillis() - c.getIdleStartMilliseconds(); if (idleMilliseconds > 0) { LOGGER.info("Idle timeout: "+c.getName()); LOGGER.log(Level.INFO, "Terminating " + c); - try { - c.getNode().retentionTerminate(); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } + c.getNode().retentionTerminate(); } } return 1; @@ -48,6 +41,11 @@ public void start(DockerComputer c) { c.connect(false); } + @Override + public boolean isManualLaunchAllowed(DockerComputer c) { + return false; + } + // no registration since this retention strategy is used only for EC2 nodes that we provision automatically. // @Extension public static class DescriptorImpl extends Descriptor> { diff --git a/src/main/java/com/nirima/jenkins/plugins/docker/DockerSlave.java b/src/main/java/com/nirima/jenkins/plugins/docker/DockerSlave.java index 6c2a0f8d4..ee79b1c66 100644 --- a/src/main/java/com/nirima/jenkins/plugins/docker/DockerSlave.java +++ b/src/main/java/com/nirima/jenkins/plugins/docker/DockerSlave.java @@ -1,21 +1,39 @@ package com.nirima.jenkins.plugins.docker; -import com.google.common.base.Objects; -import com.kpelykh.docker.client.DockerClient; -import com.kpelykh.docker.client.DockerException; -import com.kpelykh.docker.client.model.CommitConfig; -import com.nirima.jenkins.plugins.docker.action.DockerBuildAction; -import hudson.model.*; +import hudson.Extension; +import hudson.model.Messages; +import hudson.model.TaskListener; +import hudson.model.Computer; +import hudson.model.Descriptor; +import hudson.model.Label; +import hudson.model.Node.Mode; +import hudson.model.Queue; +import hudson.model.Run; +import hudson.model.queue.CauseOfBlockage; import hudson.slaves.AbstractCloudSlave; -import hudson.slaves.ComputerLauncher; import hudson.slaves.NodeProperty; +import hudson.slaves.ComputerLauncher; import hudson.slaves.RetentionStrategy; +import hudson.triggers.SafeTimerTask; +import java.io.File; import java.io.IOException; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import jenkins.model.Jenkins; +import jenkins.util.Timer; + +import org.acegisecurity.Authentication; +import org.apache.commons.io.FileUtils; + +import com.google.common.base.Objects; +import com.github.dockerjava.client.DockerClient; +import com.github.dockerjava.client.DockerException; +import com.github.dockerjava.client.model.CommitConfig; +import com.nirima.jenkins.plugins.docker.action.DockerBuildAction; + public class DockerSlave extends AbstractCloudSlave { @@ -33,6 +51,34 @@ public DockerSlave(DockerTemplate dockerTemplate, String containerId, String nam this.containerId = containerId; } + /** + * Overriding because of a bug in Jenkins that prevents Slaves from being EXCLUSIVE while the Master has a number of + * executors = 0. + */ + @Override + public CauseOfBlockage canTake(Queue.BuildableItem item) { + Label l = item.getAssignedLabel(); + if(l!=null && !l.contains(this)) + return CauseOfBlockage.fromMessage(Messages._Node_LabelMissing(getNodeName(),l)); // the task needs to be executed on label that this node doesn't have. + + Authentication identity = item.authenticate(); + if (!getACL().hasPermission(identity,Computer.BUILD)) { + // doesn't have a permission + // TODO: does it make more sense to define a separate permission? + return CauseOfBlockage.fromMessage(Messages._Node_LackingBuildPermission(identity.getName(),getNodeName())); + } + + // Check each NodeProperty to see whether they object to this node + // taking the task + for (NodeProperty prop: getNodeProperties()) { + CauseOfBlockage c = prop.canTake(item); + if (c!=null) return c; + } + + // Looks like we can take the task + return null; + } + public DockerCloud getCloud() { return dockerTemplate.getParent(); } @@ -56,33 +102,56 @@ protected void _terminate(TaskListener listener) throws IOException, Interrupted DockerClient client = getClient(); try { + LOGGER.log(Level.INFO, "Disconnecting slave " + super.getDisplayName()); toComputer().disconnect(null); - client.stopContainer(containerId); - - if( theRun != null ) { + try { + client.stopContainerCmd(containerId).exec(); + + if (theRun != null) { + try { + commit(); + } + catch (DockerException e) { + LOGGER.log(Level.SEVERE, "Failure to commit Docker container " + containerId, e); + } + } + try { - commit(); - } catch (DockerException e) { - LOGGER.log(Level.SEVERE, "Failure to commit instance " + containerId); + client.removeContainerCmd(containerId).exec(); + } + catch (DockerException e) { + LOGGER.log(Level.SEVERE, "Failure to remove container " + containerId, e); } } - - client.removeContainer(containerId); - } catch (DockerException e) { - LOGGER.log(Level.SEVERE, "Failure to terminate instance " + containerId); + catch (DockerException e) { + LOGGER.log(Level.SEVERE, "Failure to stop container " + containerId, e); + } + } + catch (Exception e) { + LOGGER.log(Level.SEVERE, "Failure to disconnect, stop or remove Docker container " + containerId, e); + } + finally { + try { + // delete log directory/files + File slaveLogDir = new File(Jenkins.getInstance().getRootDir(), "logs/slaves/" + getDisplayName()); + if (slaveLogDir.exists()) { + FileUtils.deleteDirectory(slaveLogDir); + } + } + finally { + dockerTemplate.containerTerminated(this, listener); + } } } public void commit() throws DockerException, IOException { DockerClient client = getClient(); - CommitConfig commitConfig = new CommitConfig.Builder(containerId) - .author("Jenkins") - .repo(theRun.getParent().getDisplayName()) - .tag(theRun.getDisplayName()) - .build(); - - String tag_image = client.commit(commitConfig); + String tag_image = client.commitCmd(containerId) + .withAuthor("Jenkins") + .withRepository(theRun.getParent().getDisplayName()) + .withTag(theRun.getDisplayName()) + .exec(); theRun.addAction( new DockerBuildAction(getCloud().serverUrl, containerId, tag_image) ); theRun.save(); @@ -96,11 +165,22 @@ public DockerClient getClient() { * Called when the slave is connected to Jenkins */ public void onConnected() { - + LOGGER.info("Docker provisioned slave " + getDisplayName() + " connected"); } - public void retentionTerminate() throws IOException, InterruptedException { - terminate(); + public void retentionTerminate() { + Timer.get().submit(new SafeTimerTask() { + public void doRun() { + try { + LOGGER.log(Level.INFO, "Terminating Docker provisioned slave " + getDisplayName()); + terminate(); + LOGGER.log(Level.INFO, "Terminated Docker provisioned slave " + getDisplayName()); + } + catch (Exception e) { + LOGGER.log(Level.WARNING, "Error terminating Docker provisioned slave " + getDisplayName(), e); + } + } + }); } @Override @@ -109,4 +189,19 @@ public String toString() { .add("containerId", containerId) .toString(); } + + @Extension + public static final class DescriptorImpl extends SlaveDescriptor { + + @Override + public String getDisplayName() { + return "Docker Slave"; + }; + + @Override + public boolean isInstantiable() { + return false; + } + + } } diff --git a/src/main/java/com/nirima/jenkins/plugins/docker/DockerTemplate.java b/src/main/java/com/nirima/jenkins/plugins/docker/DockerTemplate.java index 16a670f57..8c28f5ceb 100644 --- a/src/main/java/com/nirima/jenkins/plugins/docker/DockerTemplate.java +++ b/src/main/java/com/nirima/jenkins/plugins/docker/DockerTemplate.java @@ -1,39 +1,56 @@ package com.nirima.jenkins.plugins.docker; -import com.cloudbees.jenkins.plugins.sshcredentials.SSHAuthenticator; -import com.cloudbees.jenkins.plugins.sshcredentials.SSHUserListBoxModel; -import com.cloudbees.plugins.credentials.CredentialsProvider; -import com.cloudbees.plugins.credentials.common.StandardUsernameCredentials; -import com.cloudbees.plugins.credentials.domains.HostnamePortRequirement; -import com.google.common.base.Strings; -import com.kpelykh.docker.client.DockerClient; -import com.kpelykh.docker.client.DockerException; -import com.kpelykh.docker.client.model.*; -import com.trilead.ssh2.Connection; import hudson.Extension; import hudson.Util; -import hudson.model.*; +import hudson.model.Describable; +import hudson.model.ItemGroup; +import hudson.model.TaskListener; +import hudson.model.Descriptor; +import hudson.model.Label; +import hudson.model.Node; import hudson.model.labels.LabelAtom; import hudson.plugins.sshslaves.SSHLauncher; -import hudson.remoting.Channel; import hudson.security.ACL; -import hudson.slaves.ComputerLauncher; import hudson.slaves.NodeProperty; +import hudson.slaves.NodePropertyDescriptor; +import hudson.slaves.ComputerLauncher; import hudson.slaves.RetentionStrategy; -import hudson.util.ListBoxModel; +import hudson.util.DescribableList; import hudson.util.StreamTaskListener; -import jenkins.model.Jenkins; -import org.kohsuke.stapler.AncestorInPath; -import org.kohsuke.stapler.DataBoundConstructor; -import org.kohsuke.stapler.QueryParameter; +import hudson.util.ListBoxModel; import java.io.IOException; import java.io.PrintStream; -import java.net.URL; -import java.util.*; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; +import jenkins.model.Jenkins; +import org.apache.commons.lang.StringUtils; +import org.kohsuke.stapler.AncestorInPath; +import org.kohsuke.stapler.DataBoundConstructor; + +import com.cloudbees.jenkins.plugins.sshcredentials.SSHAuthenticator; +import com.cloudbees.jenkins.plugins.sshcredentials.SSHUserListBoxModel; +import com.cloudbees.plugins.credentials.CredentialsProvider; +import com.cloudbees.plugins.credentials.common.StandardUsernameCredentials; +import com.google.common.base.Strings; +import com.github.dockerjava.client.DockerClient; +import com.github.dockerjava.client.DockerException; +import com.github.dockerjava.client.model.ContainerConfig; +import com.github.dockerjava.client.model.ContainerCreateResponse; +import com.github.dockerjava.client.model.ContainerInspectResponse; +import com.github.dockerjava.client.model.ExposedPort; +import com.github.dockerjava.client.model.HostConfig; +import com.github.dockerjava.client.model.Ports; +import com.github.dockerjava.client.model.Ports.Binding; +import com.trilead.ssh2.Connection; + +/** + * This is not a Node. It only extends Node for Node-scoped tools. + */ public class DockerTemplate implements Describable { private static final Logger LOGGER = Logger.getLogger(DockerTemplate.class.getName()); @@ -71,11 +88,16 @@ public class DockerTemplate implements Describable { public final String remoteFs; // = "/home/jenkins"; - public final int instanceCap; + public final int instanceCap; // maximum number of containers allowed to be created at one time + public final int minIdleContainers; // minimum number of containers to have waiting for jobs public final boolean tagOnCompletion; + public final int sshPort; + + private /*almost final*/ DescribableList,NodePropertyDescriptor> nodeProperties = new DescribableList,NodePropertyDescriptor>(Jenkins.getInstance()); + private transient /*almost final*/ Set labelSet; public transient DockerCloud parent; @@ -84,8 +106,9 @@ public DockerTemplate(String image, String labelString, String remoteFs, String credentialsId, String jvmOptions, String javaPath, String prefixStartSlaveCmd, String suffixStartSlaveCmd, - boolean tagOnCompletion, String instanceCapStr - ) { + boolean tagOnCompletion, String instanceCapStr, int sshPort, + List> nodeProperties, String minIdleContainersStr) + throws IOException { this.image = image; this.labelString = Util.fixNull(labelString); this.credentialsId = credentialsId; @@ -96,12 +119,26 @@ public DockerTemplate(String image, String labelString, this.remoteFs = Strings.isNullOrEmpty(remoteFs)?"/home/jenkins":remoteFs; this.tagOnCompletion = tagOnCompletion; - if(instanceCapStr.equals("")) { + if (Strings.isNullOrEmpty(instanceCapStr)) { this.instanceCap = Integer.MAX_VALUE; } else { this.instanceCap = Integer.parseInt(instanceCapStr); } + if (Strings.isNullOrEmpty(minIdleContainersStr)) { + this.minIdleContainers = 0; + } else { + this.minIdleContainers = Integer.parseInt(minIdleContainersStr); + } + + if (this.minIdleContainers > this.instanceCap) { + throw new RuntimeException("Minimum number of idle containers must be less than or equals to the container cap."); + } + + this.sshPort = sshPort; + + this.nodeProperties.replaceBy(nodeProperties); + readResolve(); } @@ -113,6 +150,18 @@ public String getInstanceCapStr() { } } + public int getInstanceCap() { + return instanceCap; + } + + public String getMinIdleContainersStr() { + return String.valueOf(minIdleContainers); + } + + public int getMinIdleContainers() { + return minIdleContainers; + } + public Descriptor getDescriptor() { return Jenkins.getInstance().getDescriptor(getClass()); } @@ -124,7 +173,7 @@ public Set getLabelSet(){ /** * Initializes data structure that we don't persist. */ - protected Object readResolve() { + public Object readResolve() { labelSet = Label.parse(labelString); return this; } @@ -140,55 +189,50 @@ public DockerCloud getParent() { public DockerSlave provision(StreamTaskListener listener) throws IOException, Descriptor.FormException, DockerException { PrintStream logger = listener.getLogger(); DockerClient dockerClient = getParent().connect(); - - logger.println("Launching " + image ); String nodeDescription = "Docker Node"; - - + int numExecutors = 1; Node.Mode mode = Node.Mode.EXCLUSIVE; - RetentionStrategy retentionStrategy = new DockerRetentionStrategy();//RetentionStrategy.INSTANCE; - List> nodeProperties = new ArrayList(); - - ContainerConfig containerConfig = new ContainerConfig(); - containerConfig.setImage(image); - containerConfig.setCmd(new String[]{"/usr/sbin/sshd", "-D"}); - containerConfig.setPortSpecs(new String[]{"22"}); - - - - ContainerCreateResponse container = dockerClient.createContainer(containerConfig); - - - // Launch it.. : - // MAybe should be in computerLauncher - - Map bports = new HashMap(); - PortBinding binding = new PortBinding(); - binding.hostIp = "0.0.0.0"; - // binding.hostPort = ""; - bports.put("22/tcp", new PortBinding[] { binding }); - - HostConfig hostConfig = new HostConfig(); - hostConfig.setPortBindings(bports); - - - dockerClient.startContainer(container.getId(), hostConfig); - + ContainerCreateResponse container = dockerClient.createContainerCmd(image) + .withCmd("/usr/sbin/sshd", "-D") + .withExposedPorts(ExposedPort.tcp(22)) + .exec(); String containerId = container.getId(); - ContainerInspectResponse containerInspectResponse = dockerClient.inspectContainer(containerId); + // Launch it.. + boolean removeContainer = true; + try { + Ports bports = new Ports(); + bports.bind(ExposedPort.tcp(22), new Binding("0.0.0.0", sshPort)); + dockerClient.startContainerCmd(containerId) + .withPortBindings(bports) + .exec(); + removeContainer = false; + } + finally { + if (removeContainer) { + try { + dockerClient.removeContainerCmd(containerId).exec(); + } + catch (DockerException e) { + LOGGER.log(Level.SEVERE, "Failure to remove container " + containerId + " that did not start.", e); + } + } + } + + ContainerInspectResponse containerInspectResponse = dockerClient.inspectContainerCmd(containerId).exec(); ComputerLauncher launcher = new DockerComputerLauncher(this, containerInspectResponse); + String nodeName = this.image + "-" + containerId.substring(0, 12); return new DockerSlave(this, containerId, - containerId.substring(12), + nodeName, nodeDescription, remoteFs, numExecutors, mode, labelString, launcher, retentionStrategy, nodeProperties); @@ -199,6 +243,15 @@ public int getNumExecutors() { return 1; } + public int getSshPort() { + return sshPort; + } + + public DescribableList, NodePropertyDescriptor> getNodeProperties() { + assert nodeProperties != null; + return nodeProperties; + } + @Extension public static final class DescriptorImpl extends Descriptor { @@ -208,10 +261,13 @@ public String getDisplayName() { } public ListBoxModel doFillCredentialsIdItems(@AncestorInPath ItemGroup context) { - return new SSHUserListBoxModel().withMatching(SSHAuthenticator.matcher(Connection.class), CredentialsProvider.lookupCredentials(StandardUsernameCredentials.class, context, ACL.SYSTEM, SSHLauncher.SSH_SCHEME)); } } + + void containerTerminated(DockerSlave dockerSlave, TaskListener listener) { + parent.containerTerminated(this, dockerSlave, listener); + } } diff --git a/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/config.jelly b/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/config.jelly index 75f2518e4..11a98e5e3 100644 --- a/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/config.jelly +++ b/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/config.jelly @@ -4,6 +4,6 @@ - + diff --git a/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/configure-entries.jelly b/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/configure-entries.jelly new file mode 100644 index 000000000..06a7c279f --- /dev/null +++ b/src/main/resources/com/nirima/jenkins/plugins/docker/DockerSlave/configure-entries.jelly @@ -0,0 +1,3 @@ + + + diff --git a/src/main/resources/com/nirima/jenkins/plugins/docker/DockerTemplate/config.jelly b/src/main/resources/com/nirima/jenkins/plugins/docker/DockerTemplate/config.jelly index 7f0c483b3..03b7a6f5b 100644 --- a/src/main/resources/com/nirima/jenkins/plugins/docker/DockerTemplate/config.jelly +++ b/src/main/resources/com/nirima/jenkins/plugins/docker/DockerTemplate/config.jelly @@ -27,8 +27,16 @@ + + + + + + + + @@ -45,7 +53,9 @@ - + + +