Skip to content

Commit 71b62e8

Browse files
aozarovaozarov
committed
Merge pull request #162 from mziccard/use-installed-gcd-for-testing
Avoid downloading gcd if gcloud is installed and gcd version matches v1beta2
2 parents 34d5b7a + f753b44 commit 71b62e8

1 file changed

Lines changed: 229 additions & 32 deletions

File tree

‎gcloud-java-datastore/src/test/java/com/google/gcloud/datastore/LocalGcdHelper.java‎

Lines changed: 229 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -40,40 +40,132 @@
4040
import java.nio.channels.ReadableByteChannel;
4141
import java.nio.file.FileVisitResult;
4242
import java.nio.file.Files;
43+
import java.nio.file.InvalidPathException;
4344
import java.nio.file.Path;
4445
import java.nio.file.Paths;
4546
import java.nio.file.SimpleFileVisitor;
4647
import java.nio.file.attribute.BasicFileAttributes;
4748
import java.security.MessageDigest;
4849
import java.security.NoSuchAlgorithmException;
50+
import java.util.ArrayList;
51+
import java.util.Arrays;
52+
import java.util.List;
4953
import java.util.Locale;
54+
import java.util.regex.Pattern;
5055
import java.util.zip.ZipEntry;
5156
import java.util.zip.ZipInputStream;
5257

58+
import java.util.logging.Level;
59+
import java.util.logging.Logger;
60+
5361
/**
5462
* Utility to start and stop local Google Cloud Datastore process.
5563
*/
5664
public class LocalGcdHelper {
5765

66+
private static final Logger log = Logger.getLogger(LocalGcdHelper.class.getName());
67+
5868
private final String projectId;
5969
private Path gcdPath;
6070
private ProcessStreamReader processReader;
6171

6272
public static final String DEFAULT_PROJECT_ID = "projectid1";
6373
public static final int PORT = 8080;
64-
private static final String GCD = "gcd-v1beta2-rev1-2.1.2b";
65-
private static final String GCD_FILENAME = GCD + ".zip";
74+
private static final String GCD_VERSION = "v1beta2";
75+
private static final String GCD_BUILD = "rev1-2.1.2b";
76+
private static final String GCD_BASENAME = "gcd-" + GCD_VERSION + "-" + GCD_BUILD;
77+
private static final String GCD_FILENAME = GCD_BASENAME + ".zip";
6678
private static final String MD5_CHECKSUM = "d84384cdfa8658e1204f4f8be51300e8";
6779
private static final URL GCD_URL;
80+
private static final String GCLOUD = "gcloud";
81+
private static final Path INSTALLED_GCD_PATH;
82+
private static final String GCD_VERSION_PREFIX = "gcd-emulator ";
6883

6984
static {
70-
try {
71-
GCD_URL = new URL("http://storage.googleapis.com/gcd/tools/" + GCD_FILENAME);
72-
} catch (MalformedURLException e) {
73-
throw new RuntimeException(e);
85+
INSTALLED_GCD_PATH = installedGcdPath();
86+
if (INSTALLED_GCD_PATH != null) {
87+
GCD_URL = null;
88+
} else {
89+
try {
90+
GCD_URL = new URL("http://storage.googleapis.com/gcd/tools/" + GCD_FILENAME);
91+
} catch (MalformedURLException e) {
92+
throw new RuntimeException(e);
93+
}
94+
}
95+
}
96+
97+
private static Path installedGcdPath() {
98+
String gcloudExecutableName;
99+
if (isWindows()) {
100+
gcloudExecutableName = GCLOUD + ".cmd";
101+
} else {
102+
gcloudExecutableName = GCLOUD;
103+
}
104+
Path gcloudPath = executablePath(gcloudExecutableName);
105+
gcloudPath = (gcloudPath == null) ? null : gcloudPath.getParent();
106+
if (gcloudPath == null) {
107+
if (log.isLoggable(Level.FINE)) {
108+
log.fine("SDK not found");
109+
}
110+
return null;
111+
}
112+
if (log.isLoggable(Level.FINE)) {
113+
log.fine("SDK found, looking for datastore emulator");
114+
}
115+
Path installedGcdPath = gcloudPath.resolve("platform").resolve("gcd");
116+
if (Files.exists(installedGcdPath)) {
117+
try {
118+
String installedVersion = installedGcdVersion();
119+
if (installedVersion != null && installedVersion.startsWith(GCD_VERSION)) {
120+
if (log.isLoggable(Level.FINE)) {
121+
log.fine("SDK datastore emulator found");
122+
}
123+
return installedGcdPath;
124+
} else {
125+
if (log.isLoggable(Level.FINE)) {
126+
log.fine("SDK datastore emulator found but version mismatch");
127+
}
128+
}
129+
} catch (IOException | InterruptedException ignore) {
130+
// ignore
131+
}
132+
}
133+
return null;
134+
}
135+
136+
private static String installedGcdVersion() throws IOException, InterruptedException {
137+
Process process =
138+
CommandWrapper.create().command("gcloud", "version").redirectErrorStream().start();
139+
process.waitFor();
140+
try (BufferedReader reader =
141+
new BufferedReader(new InputStreamReader(process.getInputStream()))) {
142+
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
143+
if (line.startsWith(GCD_VERSION_PREFIX)) {
144+
String[] lineComponents = line.split(" ");
145+
if (lineComponents.length > 1) {
146+
return lineComponents[1];
147+
}
148+
}
149+
}
150+
return null;
74151
}
75152
}
76153

154+
private static Path executablePath(String cmd) {
155+
String[] paths = System.getenv("PATH").split(Pattern.quote(File.pathSeparator));
156+
for (String pathString : paths) {
157+
try {
158+
Path path = Paths.get(pathString);
159+
if (Files.exists(path.resolve(cmd))) {
160+
return path;
161+
}
162+
} catch (InvalidPathException ignore) {
163+
// ignore
164+
}
165+
}
166+
return null;
167+
}
168+
77169
private static class ProcessStreamReader extends Thread {
78170

79171
private final Process process;
@@ -83,7 +175,7 @@ private static class ProcessStreamReader extends Thread {
83175
super("Local GCD InputStream reader");
84176
setDaemon(true);
85177
this.process = process;
86-
reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
178+
reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
87179
if (!Strings.isNullOrEmpty(blockUntil)) {
88180
String line;
89181
do {
@@ -116,6 +208,81 @@ public static ProcessStreamReader start(Process process, String blockUntil) thro
116208
}
117209
}
118210

211+
private static class CommandWrapper {
212+
213+
private final List<String> prefix;
214+
private List<String> command;
215+
private String nullFilename;
216+
private boolean redirectOutputToNull;
217+
private boolean redirectErrorStream;
218+
private boolean redirectErrorInherit;
219+
private Path directory;
220+
221+
private CommandWrapper() {
222+
this.prefix = new ArrayList<>();
223+
if (isWindows()) {
224+
this.prefix.add("cmd");
225+
this.prefix.add("/C");
226+
this.nullFilename = "NUL:";
227+
} else {
228+
this.prefix.add("bash");
229+
this.nullFilename = "/dev/null";
230+
}
231+
}
232+
233+
public CommandWrapper command(String... command) {
234+
this.command = new ArrayList<>(command.length + this.prefix.size());
235+
this.command.addAll(prefix);
236+
this.command.addAll(Arrays.asList(command));
237+
return this;
238+
}
239+
240+
public CommandWrapper redirectOutputToNull() {
241+
this.redirectOutputToNull = true;
242+
return this;
243+
}
244+
245+
public CommandWrapper redirectErrorStream() {
246+
this.redirectErrorStream = true;
247+
return this;
248+
}
249+
250+
public CommandWrapper redirectErrorInherit() {
251+
this.redirectErrorInherit = true;
252+
return this;
253+
}
254+
255+
public CommandWrapper directory(Path directory) {
256+
this.directory = directory;
257+
return this;
258+
}
259+
260+
public ProcessBuilder builder() {
261+
ProcessBuilder builder = new ProcessBuilder(command);
262+
if (redirectOutputToNull) {
263+
builder.redirectOutput(new File(nullFilename));
264+
}
265+
if (directory != null) {
266+
builder.directory(directory.toFile());
267+
}
268+
if (redirectErrorStream) {
269+
builder.redirectErrorStream(true);
270+
}
271+
if (redirectErrorInherit) {
272+
builder.redirectError(ProcessBuilder.Redirect.INHERIT);
273+
}
274+
return builder;
275+
}
276+
277+
public Process start() throws IOException {
278+
return builder().start();
279+
}
280+
281+
public static CommandWrapper create() {
282+
return new CommandWrapper();
283+
}
284+
}
285+
119286
public LocalGcdHelper(String projectId) {
120287
this.projectId = projectId;
121288
}
@@ -136,19 +303,41 @@ public void start() throws IOException, InterruptedException {
136303
File gcdFolder = gcdPath.toFile();
137304
gcdFolder.deleteOnExit();
138305

306+
Path gcdExecutablePath;
307+
// If cloud is available we use it, otherwise we download and start gcd
308+
if (INSTALLED_GCD_PATH == null) {
309+
downloadGcd();
310+
gcdExecutablePath = gcdPath.resolve(GCD_BASENAME);
311+
} else {
312+
gcdExecutablePath = INSTALLED_GCD_PATH;
313+
}
314+
startGcd(gcdExecutablePath);
315+
}
316+
317+
private void downloadGcd() throws IOException {
139318
// check if we already have a local copy of the gcd utility and download it if not.
140319
File gcdZipFile = new File(System.getProperty("java.io.tmpdir"), GCD_FILENAME);
141320
if (!gcdZipFile.exists() || !MD5_CHECKSUM.equals(md5(gcdZipFile))) {
321+
if (log.isLoggable(Level.FINE)) {
322+
log.fine("Fetching datastore emulator");
323+
}
142324
ReadableByteChannel rbc = Channels.newChannel(GCD_URL.openStream());
143325
try (FileOutputStream fos = new FileOutputStream(gcdZipFile)) {
144326
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
145327
}
328+
} else {
329+
if (log.isLoggable(Level.FINE)) {
330+
log.fine("Using cached datastore emulator");
331+
}
146332
}
147333
// unzip the gcd
148334
try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(gcdZipFile))) {
335+
if (log.isLoggable(Level.FINE)) {
336+
log.fine("Unzipping datastore emulator");
337+
}
149338
ZipEntry entry = zipIn.getNextEntry();
150339
while (entry != null) {
151-
File filePath = new File(gcdFolder, entry.getName());
340+
File filePath = new File(gcdPath.toFile(), entry.getName());
152341
if (!entry.isDirectory()) {
153342
extractFile(zipIn, filePath);
154343
} else {
@@ -158,38 +347,46 @@ public void start() throws IOException, InterruptedException {
158347
entry = zipIn.getNextEntry();
159348
}
160349
}
350+
}
351+
352+
private void startGcd(Path executablePath) throws IOException, InterruptedException {
161353
// cleanup any possible data for the same project
162-
File datasetFolder = new File(gcdFolder, GCD + '/' + projectId);
354+
File datasetFolder = new File(gcdPath.toFile(), projectId);
163355
deleteRecurse(datasetFolder.toPath());
164356

165-
// create the datastore for the project
166-
ProcessBuilder processBuilder = new ProcessBuilder()
167-
.redirectError(ProcessBuilder.Redirect.INHERIT)
168-
.directory(new File(gcdFolder, GCD));
357+
// Get path to cmd executable
358+
Path gcdAbsolutePath;
169359
if (isWindows()) {
170-
processBuilder.command("cmd", "/C", "gcd.cmd", "create", "-p", projectId, projectId);
171-
processBuilder.redirectOutput(new File("NULL:"));
360+
gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.cmd");
172361
} else {
173-
processBuilder.redirectOutput(new File("/dev/null"));
174-
processBuilder.command("bash", "gcd.sh", "create", "-p", projectId, projectId);
362+
gcdAbsolutePath = executablePath.toAbsolutePath().resolve("gcd.sh");
175363
}
176364

177-
Process temp = processBuilder.start();
178-
temp.waitFor();
365+
// create the datastore for the project
366+
if (log.isLoggable(Level.FINE)) {
367+
log.log(Level.FINE, "Creating datastore for the project: {0}", projectId);
368+
}
369+
Process createProcess =
370+
CommandWrapper.create()
371+
.command(gcdAbsolutePath.toString(), "create", "-p", projectId, projectId)
372+
.redirectErrorInherit()
373+
.directory(gcdPath)
374+
.redirectOutputToNull()
375+
.start();
376+
createProcess.waitFor();
179377

180378
// start the datastore for the project
181-
processBuilder = new ProcessBuilder()
182-
.directory(new File(gcdFolder, GCD))
183-
.redirectErrorStream(true);
184-
if (isWindows()) {
185-
processBuilder.command("cmd", "/C", "gcd.cmd", "start", "--testing",
186-
"--allow_remote_shutdown", projectId);
187-
} else {
188-
processBuilder.command("bash", "gcd.sh", "start", "--testing", "--allow_remote_shutdown",
189-
projectId);
379+
if (log.isLoggable(Level.FINE)) {
380+
log.log(Level.FINE, "Starting datastore emulator for the project: {0}", projectId);
190381
}
191-
temp = processBuilder.start();
192-
processReader = ProcessStreamReader.start(temp, "Dev App Server is now running");
382+
Process startProcess =
383+
CommandWrapper.create()
384+
.command(gcdAbsolutePath.toString(), "start", "--testing", "--allow_remote_shutdown",
385+
projectId)
386+
.directory(gcdPath)
387+
.redirectErrorStream()
388+
.start();
389+
processReader = ProcessStreamReader.start(startProcess, "Dev App Server is now running");
193390
}
194391

195392
private static String md5(File gcdZipFile) throws IOException {
@@ -202,7 +399,7 @@ private static String md5(File gcdZipFile) throws IOException {
202399
md5.update(bytes, 0, len);
203400
}
204401
}
205-
return String.format("%032x",new BigInteger(1, md5.digest()));
402+
return String.format("%032x", new BigInteger(1, md5.digest()));
206403
} catch (NoSuchAlgorithmException e) {
207404
throw new IOException(e);
208405
}
@@ -312,7 +509,7 @@ public static boolean isActive(String projectId) {
312509
urlBuilder.append("/datastore/v1beta2/datasets/").append(projectId).append("/lookup");
313510
URL url = new URL(urlBuilder.toString());
314511
try (BufferedReader reader =
315-
new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) {
512+
new BufferedReader(new InputStreamReader(url.openStream(), UTF_8))) {
316513
return "Valid RPC".equals(reader.readLine());
317514
}
318515
} catch (IOException ignore) {

0 commit comments

Comments
 (0)