diff --git a/.travis.yml b/.travis.yml index 48bfef593c177..e1fe46f9a88eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,11 @@ addons: apt: packages: - oracle-java8-installer + - oracle-java8-set-default + +env: + global: + - JAVA_HOME=/usr/lib/jvm/java-8-oracle install: - travis_wait 20 make install diff --git a/Makefile b/Makefile index a6ccd3eab9731..70fe3ed83e9e7 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ compile: ## Compile Java code (KCL library utils) echo "Compiling" $(VENV_RUN); python -c 'from localstack.utils.kinesis import kclipy_helper; print kclipy_helper.get_kcl_classpath()' javac -cp $(shell $(VENV_RUN); python -c 'from localstack.utils.kinesis import kclipy_helper; print kclipy_helper.get_kcl_classpath()') localstack/utils/kinesis/java/com/atlassian/*.java + (test ! -e ext/java || cd ext/java && mvn -DskipTests package) # TODO enable once we want to support Java-based Lambdas # (cd localstack/mock && mvn package) @@ -39,7 +40,7 @@ coveralls: ## Publish coveralls metrics ($(VENV_RUN); coveralls) infra: ## Manually start the local infrastructure for testing - ($(VENV_RUN); localstack/mock/infra.py) + $(VENV_RUN); exec localstack/mock/infra.py web: ## Start web application (dashboard) ($(VENV_RUN); bin/localstack web --port=8081) diff --git a/README.md b/README.md index 53baf9b413505..360132c06b0e1 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,17 @@ missing functionality on top of them: * `npm` (node.js package manager) * `java`/`javac` (Java runtime environment and compiler) -## Installation +## Installing -To install the tool and all its dependencies, run the following command: +The easiest way to install *LocalStack* is via `pip`: + +``` +pip install localstack +``` + +## Developing + +If you pull the repo in order to extend/modify LocalStack, run this command to install all dependencies: ``` make install @@ -131,6 +139,27 @@ def my_app_test(): See the example test file `tests/test_integration.py` for more details. +## Integration with Java/JUnit + +In order to use *LocalStack* with Java, the project ships with a simple JUnit runner. Take a look +at the example JUnit test in `ext/java`. When you run the test, all dependencies are automatically +downloaded and installed to a temporary directory in your system. + +``` +@RunWith(LocalstackTestRunner.class) +public class MyCloudAppTest { + + @Test + public void testLocalS3API() { + AmazonS3 s3 = new AmazonS3Client(...); + s3.setEndpoint(LocalstackTestRunner.getEndpointS3()); + List buckets = s3.listBuckets(); + ... + } + +} +``` + ## Web Dashboard The projects also comes with a simple Web dashboard that allows to view the @@ -143,6 +172,8 @@ make web ## Change Log +* v0.3.0: Add simple integration for JUnit; improve process signal handling +* v0.2.11: Refactored the AWS assume role function * v0.2.10: Added AWS assume role functionality. * v0.2.9: Kinesis error response formatting * v0.2.7: Throw Kinesis errors randomly diff --git a/ext/java/.gitignore b/ext/java/.gitignore new file mode 100644 index 0000000000000..b83d22266ac8a --- /dev/null +++ b/ext/java/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/ext/java/pom.xml b/ext/java/pom.xml new file mode 100644 index 0000000000000..df316963f0374 --- /dev/null +++ b/ext/java/pom.xml @@ -0,0 +1,55 @@ + + 4.0.0 + + com.atlassian + localstack-utils + jar + 1.0-SNAPSHOT + localstack-utils + + + + com.amazonaws + aws-lambda-java-core + 1.1.0 + + + com.amazonaws + aws-lambda-java-events + 1.3.0 + + + junit + junit + 4.12 + + + com.amazonaws + aws-java-sdk + 1.11.86 + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + false + + + + package + + shade + + + + + + + diff --git a/localstack/mock/src/main/java/com/atlassian/LambdaContext.java b/ext/java/src/main/java/com/atlassian/localstack/LambdaContext.java similarity index 97% rename from localstack/mock/src/main/java/com/atlassian/LambdaContext.java rename to ext/java/src/main/java/com/atlassian/localstack/LambdaContext.java index 3b14e1dda3ca3..3240516881cf0 100644 --- a/localstack/mock/src/main/java/com/atlassian/LambdaContext.java +++ b/ext/java/src/main/java/com/atlassian/localstack/LambdaContext.java @@ -1,4 +1,4 @@ -package com.atlassian; +package com.atlassian.localstack; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/localstack/mock/src/main/java/com/atlassian/LambdaExecutor.java b/ext/java/src/main/java/com/atlassian/localstack/LambdaExecutor.java similarity index 94% rename from localstack/mock/src/main/java/com/atlassian/LambdaExecutor.java rename to ext/java/src/main/java/com/atlassian/localstack/LambdaExecutor.java index 30993bed4ac77..3f152b9781bd8 100644 --- a/localstack/mock/src/main/java/com/atlassian/LambdaExecutor.java +++ b/ext/java/src/main/java/com/atlassian/localstack/LambdaExecutor.java @@ -1,4 +1,4 @@ -package com.atlassian; +package com.atlassian.localstack; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -21,6 +21,11 @@ import com.amazonaws.services.lambda.runtime.events.KinesisEvent.Record; import com.fasterxml.jackson.databind.ObjectMapper; +/** + * TODO: Support for AWS Lambda functions written in Java is work in progress. + * + * @author Waldemar Hummer + */ public class LambdaExecutor { @SuppressWarnings("unchecked") @@ -32,8 +37,8 @@ public static void main(String[] args) throws Exception { if(test) { final String testFile = "/tmp/test.event.kinesis.json"; String content = "{\"records\": [" - + "{\"kinesis\": " + - + "{}" + + + "{\"kinesis\": " + + "{}" + "}" + "]}"; writeFile(testFile, content); @@ -53,6 +58,7 @@ public void run() { KinesisEvent event = new KinesisEvent(); ObjectMapper reader = new ObjectMapper(); String fileContent = readFile(args[1]); + @SuppressWarnings("deprecation") Map map = reader.reader(Map.class).readValue(fileContent); List> records = (List>) get(map, "Records"); event.setRecords(new LinkedList()); diff --git a/ext/java/src/main/java/com/atlassian/localstack/LocalstackTestRunner.java b/ext/java/src/main/java/com/atlassian/localstack/LocalstackTestRunner.java new file mode 100644 index 0000000000000..022b7b8708a33 --- /dev/null +++ b/ext/java/src/main/java/com/atlassian/localstack/LocalstackTestRunner.java @@ -0,0 +1,190 @@ +package com.atlassian.localstack; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.model.InitializationError; + +import com.amazonaws.util.IOUtils; + +/** + * Simple JUnit test runner that automatically downloads, installs, starts, + * and stops the LocalStack local cloud infrastructure components. + * + * Should work cross-OS, however has been only tested under Unix (Linux/MacOS). + * + * @author Waldemar Hummer + */ +public class LocalstackTestRunner extends BlockJUnit4ClassRunner { + + private static final AtomicReference INFRA_STARTED = new AtomicReference(); + private static String CONFIG_FILE_CONTENT = ""; + + private static final String INFRA_READY_MARKER = "Ready."; + private static final String TMP_INSTALL_DIR = System.getProperty("java.io.tmpdir") + + File.separator + "localstack_install_dir"; + private static final String ADDITIONAL_PATH = "/usr/local/bin/"; + private static final String LOCALHOST = "localhost"; + private static final String LOCALSTACK_REPO_URL = "https://github.com/atlassian/localstack"; + + private static final Logger LOG = Logger.getLogger(LocalstackTestRunner.class.getName()); + + public LocalstackTestRunner(Class klass) throws InitializationError { + super(klass); + } + + /* SERVICE ENDPOINTS */ + + public static String getEndpointS3() { + ensureInstallation(); + return getEndpoint("s3"); + } + + public static String getEndpointKinesis() { + ensureInstallation(); + return getEndpoint("kinesis"); + } + + public static String getEndpointLambda() { + ensureInstallation(); + return getEndpoint("lambda"); + } + + public static String getEndpointDynamoDB() { + ensureInstallation(); + return getEndpoint("dynamodb"); + } + + public static String getEndpointDynamoDBStreams() { + ensureInstallation(); + return getEndpoint("dynamodbstreams"); + } + + public static String getEndpointAPIGateway() { + ensureInstallation(); + return getEndpoint("apigateway"); + } + + public static String getEndpointElasticsearch() { + ensureInstallation(); + return getEndpoint("elasticsearch"); + } + + public static String getEndpointFirehose() { + ensureInstallation(); + return getEndpoint("firehose"); + } + + public static String getEndpointSNS() { + ensureInstallation(); + return getEndpoint("sns"); + } + + public static String getEndpointSQS() { + ensureInstallation(); + return getEndpoint("sns"); + } + + /* UTILITY METHODS */ + + @Override + public void run(RunNotifier notifier) { + setupInfrastructure(); + super.run(notifier); + } + + private static void ensureInstallation() { + File dir = new File(TMP_INSTALL_DIR); + if(!dir.exists()) { + LOG.info("Installing LocalStack to temporary directory (this might take a while): " + TMP_INSTALL_DIR); + exec("git clone " + LOCALSTACK_REPO_URL + " " + TMP_INSTALL_DIR); + exec("cd " + TMP_INSTALL_DIR + "; make install"); + } + } + + private static void killProcess(Process p) { + p.destroy(); + p.destroyForcibly(); + } + + private static String getEndpoint(String service) { + ensureInstallation(); + String regex = ".*DEFAULT_PORT_" + service.toUpperCase() + "\\s*=\\s*([0-9]+).*"; + String port = Pattern.compile(regex, Pattern.DOTALL | Pattern.MULTILINE).matcher(CONFIG_FILE_CONTENT).replaceAll("$1"); + return "http://" + LOCALHOST + ":" + port + "/"; + } + + private static Process exec(String cmd) { + return exec(cmd, true); + } + + private static Process exec(String cmd, boolean wait) { + try { + Map env = System.getenv(); + final Process p = Runtime.getRuntime().exec(new String[]{"/bin/bash", "-c", cmd}, + new String[]{"PATH=" + ADDITIONAL_PATH + ":" + env.get("PATH")}); + if (wait) { + int code = p.waitFor(); + if(code != 0) { + String stderr = IOUtils.toString(p.getErrorStream()); + String stdout = IOUtils.toString(p.getInputStream()); + throw new IllegalStateException("Failed to run command '" + cmd + "', return code " + code + + ".\nSTDOUT: " + stdout + "\nSTDERR: " + stderr); + } + } else { + /* make sure we destroy the process on JVM shutdown */ + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + killProcess(p); + } + }); + } + return p; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void setupInfrastructure() { + synchronized (INFRA_STARTED) { + ensureInstallation(); + if(INFRA_STARTED.get() != null) return; + String cmd = "cd " + TMP_INSTALL_DIR + "; exec make infra"; + Process proc; + try { + proc = exec(cmd, false); + BufferedReader r1 = new BufferedReader(new InputStreamReader(proc.getInputStream())); + String line; + LOG.info("Waiting for infrastructure to be spun up"); + while((line = r1.readLine()) != null) { + if(INFRA_READY_MARKER.equals(line)) { + break; + } + } + /* read contents of LocalStack config file */ + String configFile = TMP_INSTALL_DIR + File.separator + "localstack" + File.separator + "constants.py"; + CONFIG_FILE_CONTENT = IOUtils.toString(new FileInputStream(configFile)); + INFRA_STARTED.set(proc); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public static void teardownInfrastructure() { + Process proc = INFRA_STARTED.get(); + if(proc == null) { + return; + } + killProcess(proc); + } +} diff --git a/ext/java/src/test/java/com/atlassian/localstack/TestRunnerTest.java b/ext/java/src/test/java/com/atlassian/localstack/TestRunnerTest.java new file mode 100644 index 0000000000000..9435cb5fe9ac7 --- /dev/null +++ b/ext/java/src/test/java/com/atlassian/localstack/TestRunnerTest.java @@ -0,0 +1,64 @@ +package com.atlassian.localstack; + +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.kinesis.AmazonKinesis; +import com.amazonaws.services.kinesis.AmazonKinesisClient; +import com.amazonaws.services.kinesis.model.ListStreamsResult; +import com.amazonaws.services.lambda.AWSLambda; +import com.amazonaws.services.lambda.AWSLambdaClientBuilder; +import com.amazonaws.services.lambda.model.ListFunctionsResult; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.Bucket; + +@RunWith(LocalstackTestRunner.class) +public class TestRunnerTest { + + private static final String DEFAULT_REGION = "us-east-1"; + private static final String TEST_ACCESS_KEY = "test"; + private static final String TEST_SECRET_KEY = "test"; + private static final AWSCredentials TEST_CREDENTIALS = new BasicAWSCredentials(TEST_ACCESS_KEY, TEST_SECRET_KEY); + + static { + /* Need to disable CBOR protocol, see: + * https://github.com/mhart/kinesalite/blob/master/README.md#cbor-protocol-issues-with-the-java-sdk + */ + TestUtils.setEnv("AWS_CBOR_DISABLE", "1"); + } + + @Test + public void testLocalKinesisAPI() { + AmazonKinesis kinesis = new AmazonKinesisClient(TEST_CREDENTIALS); + kinesis.setEndpoint(LocalstackTestRunner.getEndpointKinesis()); + ListStreamsResult streams = kinesis.listStreams(); + Assert.assertNotNull(streams.getStreamNames()); + } + + @Test + public void testLocalS3API() { + AmazonS3 s3 = new AmazonS3Client(TEST_CREDENTIALS); + s3.setEndpoint(LocalstackTestRunner.getEndpointS3()); + List buckets = s3.listBuckets(); + Assert.assertNotNull(buckets); + } + + @Test + public void testLocalLambdaAPI() { + AWSLambda lambda = AWSLambdaClientBuilder.standard().withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration( + LocalstackTestRunner.getEndpointLambda(), DEFAULT_REGION)).withCredentials( + new AWSStaticCredentialsProvider(TEST_CREDENTIALS)).build(); + ListFunctionsResult functions = lambda.listFunctions(); + Assert.assertNotNull(functions.getFunctions()); + } + +} diff --git a/ext/java/src/test/java/com/atlassian/localstack/TestUtils.java b/ext/java/src/test/java/com/atlassian/localstack/TestUtils.java new file mode 100644 index 0000000000000..e2d1a9a4497a5 --- /dev/null +++ b/ext/java/src/test/java/com/atlassian/localstack/TestUtils.java @@ -0,0 +1,51 @@ +package com.atlassian.localstack; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +@SuppressWarnings("all") +public class TestUtils { + + protected static void setEnv(String key, String value) { + Map newEnv = new HashMap(); + newEnv.put(key, value); + setEnv(newEnv); + } + + protected static void setEnv(Map newEnv) { + try { + Class processEnvironmentClass = Class.forName("java.lang.ProcessEnvironment"); + Field theEnvironmentField = processEnvironmentClass.getDeclaredField("theEnvironment"); + theEnvironmentField.setAccessible(true); + Map env = (Map) theEnvironmentField.get(null); + env.putAll(newEnv); + Field theCaseInsensitiveEnvironmentField = processEnvironmentClass + .getDeclaredField("theCaseInsensitiveEnvironment"); + theCaseInsensitiveEnvironmentField.setAccessible(true); + Map cienv = (Map) theCaseInsensitiveEnvironmentField.get(null); + cienv.putAll(newEnv); + } catch (NoSuchFieldException e) { + try { + Class[] classes = Collections.class.getDeclaredClasses(); + Map env = System.getenv(); + for (Class cl : classes) { + if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) { + Field field = cl.getDeclaredField("m"); + field.setAccessible(true); + Object obj = field.get(env); + Map map = (Map) obj; + map.clear(); + map.putAll(newEnv); + } + } + } catch (Exception e2) { + e2.printStackTrace(); + } + } catch (Exception e1) { + e1.printStackTrace(); + } + } + +} diff --git a/localstack/mock/infra.py b/localstack/mock/infra.py index 8ae6de2708ecf..46ce855f033f7 100755 --- a/localstack/mock/infra.py +++ b/localstack/mock/infra.py @@ -4,6 +4,7 @@ import re import sys import time +import signal import traceback import logging import requests @@ -23,8 +24,9 @@ THIS_PATH = os.path.dirname(os.path.realpath(__file__)) ROOT_PATH = os.path.realpath(os.path.join(THIS_PATH, '..')) -# will be set to True if user hits CTRL-C -KILLED = False +# flag to indicate whether signal handlers have been set up already +SIGNAL_HANDLERS_SETUP = False +INFRA_STOPPED = False # cache table definitions - used for testing TABLE_DEFINITIONS = {} @@ -46,7 +48,22 @@ LOGGER = logging.getLogger(__name__) +def register_signal_handlers(): + global SIGNAL_HANDLERS_SETUP + if SIGNAL_HANDLERS_SETUP: + return + + # register signal handlers + def signal_handler(signal, frame): + stop_infra() + os._exit(0) + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + SIGNAL_HANDLERS_SETUP = True + + def do_run(cmd, async): + sys.stdout.flush() if async: t = ShellCommandThread(cmd) t.start() @@ -193,6 +210,9 @@ def start_lambda(port=DEFAULT_PORT_LAMBDA, async=False): def stop_infra(): + global INFRA_STOPPED + if INFRA_STOPPED: + return generic_proxy.QUIET = True common.cleanup(files=True, quiet=True) common.cleanup_resources() @@ -200,6 +220,7 @@ def stop_infra(): time.sleep(1) # TODO: optimize this (takes too long currently) # check_infra(retries=2, expect_shutdown=True) + INFRA_STOPPED = True def check_infra_kinesis(expect_shutdown=False): @@ -306,6 +327,8 @@ def start_infra(async=False, dynamodb_update_listener=None, kinesis_update_liste # set environment os.environ['AWS_REGION'] = DEFAULT_REGION os.environ['ENV'] = ENV_DEV + # register signal handlers + register_signal_handlers() # make sure AWS credentials are configured, otherwise boto3 bails on us check_aws_credentials() # install libs if not present @@ -338,8 +361,14 @@ def start_infra(async=False, dynamodb_update_listener=None, kinesis_update_liste time.sleep(3) # check that all infra components are up and running check_infra(apis=apis) + print('Ready.') + sys.stdout.flush() if not async and thread: - thread.join() + # this is a bit of an ugly hack, but we need to make sure that we + # stay in the execution context of the main thread, otherwise our + # signal handlers don't work + while True: + time.sleep(1) return thread except KeyboardInterrupt, e: print("Shutdown") @@ -536,6 +565,8 @@ def dynamodb_extract_keys(item, table_name): if __name__ == '__main__': print('Starting local dev environment. CTRL-C to quit.') + # set up logging logging.basicConfig(level=logging.WARNING) logging.getLogger('elasticsearch').setLevel(logging.ERROR) + # fire it up! start_infra() diff --git a/localstack/mock/pom.xml b/localstack/mock/pom.xml deleted file mode 100644 index 19fa0f8ea449a..0000000000000 --- a/localstack/mock/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - 4.0.0 - - com.atlassian.c360 - lambda-executor - jar - 1.0-SNAPSHOT - lambda-executor - - - - com.amazonaws - aws-lambda-java-core - 1.1.0 - - - com.amazonaws - aws-lambda-java-events - 1.3.0 - - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.3 - - false - - - - package - - shade - - - - - - - diff --git a/localstack/utils/aws/aws_stack.py b/localstack/utils/aws/aws_stack.py index 76935302c40b1..c0aa7daa10ca0 100644 --- a/localstack/utils/aws/aws_stack.py +++ b/localstack/utils/aws/aws_stack.py @@ -431,13 +431,10 @@ def assume_role(role_arn, update_global_session=True): # Using a timer to loop through the assume role function forever...... def loop_assume_role(role_arn, timeout=DEFAULT_TIMER_LOOP_SECONDS): + # Do an initial assume role + assume_role(role_arn, True) - def do_assume_role(): - # this will do it once (we ignore the return value as we default to update_global_session=True) - assume_role(role_arn, True) - - # we call it again after it completes - loop_assume_role(role_arn, timeout) - - t = Timer(timeout, do_assume_role) + # Create timer to loop through function + t = Timer(timeout, loop_assume_role, args=[role_arn, timeout]) + t.daemon = True t.start() diff --git a/setup.py b/setup.py index 4041c7a2c8818..0ed26fce14041 100755 --- a/setup.py +++ b/setup.py @@ -76,8 +76,8 @@ def run(self): setup( name='localstack', - version='0.2.10', - description='Provides an easy-to-use test/mocking framework for developing Cloud applications', + version='0.3.0', + description='An easy-to-use test/mocking framework for developing Cloud applications', author='Waldemar Hummer (Atlassian)', author_email='waldemar.hummer@gmail.com', url='https://bitbucket.org/atlassian/localstack',