From 6a08e56bc40ea52c9db4af1727285a5c926175b6 Mon Sep 17 00:00:00 2001 From: Sascha Wiedenfeld Date: Sun, 26 Jul 2020 21:33:02 +0200 Subject: [PATCH 1/3] Walk file tree in parallel --- .../core/dockerfile/Dockerfile.java | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java b/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java index 752935e19..ce837abac 100644 --- a/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java +++ b/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java @@ -4,10 +4,14 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; @@ -78,7 +82,7 @@ public Iterable getStatements() throws IOException { } Collection> optionals = Collections2.transform(dockerFileContent, - new LineTransformer()); + new LineTransformer()); return Optional.presentInstances(optionals); } @@ -100,7 +104,7 @@ public List getIgnores() throws IOException { ignores.add(pattern); } catch (GoLangFileMatchException e) { throw new DockerClientException(String.format( - "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); + "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); } } } @@ -132,7 +136,7 @@ public InputStream buildDockerFolderTar(File directory) { final String archiveNameWithOutExtension = UUID.randomUUID().toString(); dockerFolderTar = CompressArchiveUtil.archiveTARFiles(directory, filesToAdd, - archiveNameWithOutExtension); + archiveNameWithOutExtension); final FileInputStream tarInputStream = FileUtils.openInputStream(dockerFolderTar); final File tarFile = dockerFolderTar; @@ -180,7 +184,7 @@ public ScannedResult() throws IOException { if (matchingIgnorePattern != null) { throw new DockerClientException(String.format( - "Dockerfile is excluded by pattern '%s' in .dockerignore file", matchingIgnorePattern)); + "Dockerfile is excluded by pattern '%s' in .dockerignore file", matchingIgnorePattern)); } addFilesInDirectory(baseDirectory); @@ -201,23 +205,42 @@ private void addFilesInDirectory(File directory) { throw new DockerClientException("Failed to read build context directory: " + baseDirectory.getAbsolutePath()); } - if (files.length != 0) { - for (File f : files) { - if (f.isDirectory()) { - addFilesInDirectory(f); - } else if (effectiveMatchingIgnorePattern(f) == null) { - filesToAdd.add(f); - } + try (final Stream pathStream = Files.walk(directory.toPath()) + .parallel() + .filter(this::isNotIgnored) + .map(Path::toFile) + ) { + filesToAdd.addAll(pathStream.collect(Collectors.toList())); + } catch (IOException e) { + throw new DockerClientException("Failed to walk directory", e); + } + + } + + private boolean isNotIgnored(Path path) { + if (Files.isRegularFile(path)) { + return isEffectivelyNotIgnored(path); + } + if (Files.isDirectory(path)) { + try { + return isEmptyDirectory(path) && !isBaseDirectory(path) && isEffectivelyNotIgnored(path); + } catch (IOException e) { + throw new DockerClientException("Failed to list directory", e); } - // base directory should at least contains Dockerfile, but better check - } else if (!isBaseDirectory(directory)) { - // add empty directory - filesToAdd.add(directory); } + return false; + } + + private boolean isEmptyDirectory(Path path) throws IOException { + return Files.list(path).count() == 0; } - private boolean isBaseDirectory(File directory) { - return directory.compareTo(baseDirectory) == 0; + private boolean isEffectivelyNotIgnored(Path path) { + return effectiveMatchingIgnorePattern(path.toFile()) == null; + } + + private boolean isBaseDirectory(Path directory) { + return directory.equals(baseDirectory.toPath()); } /** @@ -236,7 +259,7 @@ private List matchingIgnorePatterns(String fileName) { } } catch (GoLangFileMatchException e) { throw new DockerClientException(String.format( - "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); + "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); } } @@ -260,6 +283,6 @@ private String effectiveMatchingIgnorePattern(File file) { String lastMatchingPattern = matchingPattern.get(matchingPattern.size() - 1); return !lastMatchingPattern.startsWith("!") ? lastMatchingPattern : null; - } + } } } From bdfea96ef8cd6b23eb9db6cf777aa031ba3a4bb3 Mon Sep 17 00:00:00 2001 From: Sascha Wiedenfeld Date: Mon, 27 Jul 2020 07:53:17 +0200 Subject: [PATCH 2/3] Relativize filename with Path java.nio.file.Path#toUri executes faster compared to java.io.File#toUri --- .../com/github/dockerjava/core/dockerfile/Dockerfile.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java b/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java index ce837abac..883b3c616 100644 --- a/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java +++ b/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java @@ -21,7 +21,6 @@ import com.github.dockerjava.core.GoLangFileMatch; import com.github.dockerjava.core.exception.GoLangFileMatchException; import com.github.dockerjava.core.util.CompressArchiveUtil; -import com.github.dockerjava.core.util.FilePathUtil; import com.google.common.base.Function; import com.google.common.base.MoreObjects; import com.google.common.base.Optional; @@ -272,7 +271,7 @@ private List matchingIgnorePatterns(String fileName) { */ private String effectiveMatchingIgnorePattern(File file) { // normalize path to replace '/' to '\' on Windows - String relativeFilename = FilenameUtils.normalize(FilePathUtil.relativize(baseDirectory, file)); + final String relativeFilename = FilenameUtils.normalize(relativizeToBaseDirectory(file)); List matchingPattern = matchingIgnorePatterns(relativeFilename); @@ -284,5 +283,9 @@ private String effectiveMatchingIgnorePattern(File file) { return !lastMatchingPattern.startsWith("!") ? lastMatchingPattern : null; } + + private String relativizeToBaseDirectory(File file) { + return baseDirectory.toPath().toUri().relativize(file.toPath().toUri()).getPath(); + } } } From 4a9cd552c5b3a42fa85f6d968d05b9a0945d0be7 Mon Sep 17 00:00:00 2001 From: Sascha Date: Mon, 27 Jul 2020 21:41:46 +0200 Subject: [PATCH 3/3] Fix whitespace formatting --- .../github/dockerjava/core/dockerfile/Dockerfile.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java b/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java index 883b3c616..dba924f7b 100644 --- a/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java +++ b/docker-java-core/src/main/java/com/github/dockerjava/core/dockerfile/Dockerfile.java @@ -81,7 +81,7 @@ public Iterable getStatements() throws IOException { } Collection> optionals = Collections2.transform(dockerFileContent, - new LineTransformer()); + new LineTransformer()); return Optional.presentInstances(optionals); } @@ -103,7 +103,7 @@ public List getIgnores() throws IOException { ignores.add(pattern); } catch (GoLangFileMatchException e) { throw new DockerClientException(String.format( - "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); + "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); } } } @@ -135,7 +135,7 @@ public InputStream buildDockerFolderTar(File directory) { final String archiveNameWithOutExtension = UUID.randomUUID().toString(); dockerFolderTar = CompressArchiveUtil.archiveTARFiles(directory, filesToAdd, - archiveNameWithOutExtension); + archiveNameWithOutExtension); final FileInputStream tarInputStream = FileUtils.openInputStream(dockerFolderTar); final File tarFile = dockerFolderTar; @@ -183,7 +183,7 @@ public ScannedResult() throws IOException { if (matchingIgnorePattern != null) { throw new DockerClientException(String.format( - "Dockerfile is excluded by pattern '%s' in .dockerignore file", matchingIgnorePattern)); + "Dockerfile is excluded by pattern '%s' in .dockerignore file", matchingIgnorePattern)); } addFilesInDirectory(baseDirectory); @@ -258,7 +258,7 @@ private List matchingIgnorePatterns(String fileName) { } } catch (GoLangFileMatchException e) { throw new DockerClientException(String.format( - "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); + "Invalid pattern '%s' on line %s in .dockerignore file", pattern, lineNumber)); } }