diff --git a/erp/com/splunk/erp/commons/ERPUtils.java b/erp/com/splunk/erp/commons/ERPUtils.java new file mode 100644 index 00000000..5dcd18e5 --- /dev/null +++ b/erp/com/splunk/erp/commons/ERPUtils.java @@ -0,0 +1,235 @@ +package com.splunk.erp.commons; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.JsonProcessingException; +import org.codehaus.jackson.map.ObjectMapper; + +import com.splunk.erp.core.IProvider; +import com.splunk.util.WildcardList; + +/** + * Helper class for ERP framework.
Contains method for parsing JsonNode and extract particular nodes for + * ProviderConfig, VixConfig, SearchInfo and WildcardList.
+ *
Get localhost information and create IProvider implementation class object. + * @author smetkar + * + */ +public class ERPUtils { + + private static Logger logger = getLogger(ERPUtils.class); + /** + * Read System.in input stream for JSON data and convert into JsonNode object + * @param System.in (InputStream) + * @return JsonNode + * @throws IOException,JsonProcessingException + */ + public static JsonNode readArgsForERP(InputStream streamToRead) throws IOException,JsonProcessingException{ + JsonNode argsAsJson = null; + String argsLine = ""; + BufferedReader reader = new BufferedReader(new InputStreamReader(streamToRead)); + argsLine = reader.readLine(); + reader.close(); + + ObjectMapper objectMapper = new ObjectMapper(); + argsAsJson = objectMapper.readTree(argsLine); + return argsAsJson; + } + + /** + * Get virtual indexes config node from JSON data + * @param JsonNode : S-2-ERP protocol JSON string + * @return JsonNode : JSON element containing Virtual Index/Indexes configuration + */ + public static JsonNode getVixesConfigNode(JsonNode argsAsJson) throws IllegalArgumentException { + JsonNode indexNode = argsAsJson.get("conf").get("indexes"); + if(indexNode == null || !(indexNode.size() > 0)) { + throw new IllegalArgumentException("Unable to find indexes element in S-2-ERP protocol JSON"); + } + return indexNode; + } + + /** + * Get provider family name + * @param JsonNode : JSON element containing family name + * @return Provider family name as String + */ + public static String getFamilyName(JsonNode providerConfigNode){ + String familyName = providerConfigNode.get("family").getValueAsText(); + return familyName; + } + + /** + * Get provider config node from JSON + * @param JsonNode : S-2-ERP protocol JSON string + * @return JsonNode : JSON element containing Provider configuration + * @throws IllegalArgumentException + */ + public static JsonNode getProviderConfigNode(JsonNode argsAsJson) throws IllegalArgumentException + { + JsonNode providerNode = argsAsJson.get("conf").get("provider"); + if(providerNode == null) { + throw new IllegalArgumentException("Unable to find provider element in S-2-ERP protocol JSON"); + } + return providerNode; + } + + /** + * Get provider name + * @param JsonNode : JSON element containing provider specific info + * @return Provider name + */ + public static String getProviderName(JsonNode providerConfigNode) { + String providerName = providerConfigNode.get("conf").get("provider").get("splunk.search.provider").getTextValue(); + return providerName; + } + + /**Get search mode, search mode can be 'stream', 'mixed' or 'report' + * @param JsonNode : JSON element containing provider specific info + * @return Search mode + */ + public static String getSearchMode(JsonNode providerConfigNode) { + String mode = providerConfigNode.get("conf").get("provider").get("mode").getTextValue(); + return mode; + } + +// /** +// * Create SearchElement object from search_expr element of S-2-ERP protocol JSON String +// * @param JsonNode : search_expr element of JSON +// * @return SearchElement object +// */ +// public static SearchElement getSearchExpression(JsonNode searchExprNode) +// { +// SearchElement element = null; +// String type = getStringParameter(searchExprNode, "type"); +// String operator = getStringParameter(searchExprNode, "op"); +// +// if(type.equalsIgnoreCase("group")) { +// ArrayList elementChildren = new ArrayList(); +// JsonNode children = searchExprNode.get("children"); +// Iterator childIterator = children.getElements(); +// while(childIterator.hasNext()) +// { +// SearchElement child = getSearchExpression(childIterator.next()); +// if(child != null) +// elementChildren.add(child); +// } +// element = new SearchGroupElement(operator, elementChildren); +// } else if(type.equalsIgnoreCase("cmp")) { +// String rhsValue = searchExprNode.get("rhs").getTextValue(); +// Object rhs = searchExprNode.get("is_numeric").getBooleanValue() ? Double.parseDouble(rhsValue) : rhsValue; +// element = new SearchCompareElement( operator, searchExprNode.get("lhs").getTextValue(), +// rhs, +// searchExprNode.get("is_negated").getBooleanValue(), +// searchExprNode.get("is_numeric").getBooleanValue(), +// searchExprNode.get("is_literal_term").getBooleanValue(), +// searchExprNode.get("is_case_sensitive").getBooleanValue(), +// searchExprNode.get("is_cidr_match").getBooleanValue()); +// } +// return element; +// } + + /** + * Creates an object of WildcardList from S-2-ERP protocol JSON string + * @param JsonNode : Json element containing required fields information + * @return Required fields list as {@link WildcardList} object + */ + public static WildcardList getRequiredFieldList(JsonNode argsAsJson) { + JsonNode requiredFieldNode = argsAsJson.get("args").get("search").get("required_fields"); + WildcardList list = null; + if(requiredFieldNode != null) { + List requiredFieldList = new ArrayList(); + if(requiredFieldNode.isArray()) { + requiredFieldList = new ArrayList(); + Iterator it = requiredFieldNode.getElements(); + while(it.hasNext()) { + requiredFieldList.add(it.next().getTextValue()); + } + } + + if(requiredFieldList.size() > 0) + list = new WildcardList(requiredFieldList); + } + return list; + } + + /** + * Determine host name + * @return String + */ + public static String getHostName() + { + String tempHostName = "localhost"; + try + { + tempHostName = InetAddress.getLocalHost().getHostName(); + }catch(UnknownHostException uhe) + { + logger.info("Could not get host name, defaulting to IP address"); + try { + tempHostName = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException unknownhHost) { + logger.info("Could not get host name, defaulting to localhost"); + } + } + return tempHostName; + } + + + /** + * Create an instance of class implementing IProvider interface + * @param String (class to be instantiated) + * @return + * @throws Exception + */ + public static IProvider getProviderInstance(String implClassName) throws Exception{ + try { + Class providerImplClass = (Class)Class.forName(implClassName); + IProvider providerImpl = providerImplClass.newInstance(); + return providerImpl; + } catch (ClassNotFoundException cnfe) { + logger.error(cnfe.getMessage(),cnfe); + throw new Exception("Error while instantiating provider object"); + } catch (InstantiationException ine) { + logger.error(ine.getMessage(),ine); + throw new Exception("Error while instantiating provider object"); + } catch (IllegalArgumentException iae) { + logger.error(iae.getMessage(),iae); + throw new Exception("Error while instantiating provider object"); + } + } + + + /** + * Get Logger for the class, default logging level is set to DEBUG + * @param Class : Class for which logger needs to be created + * @return Logger for that particular class + */ + public static Logger getLogger(Class clazz) { + Logger logger = Logger.getLogger(clazz); + + ConsoleAppender console = new ConsoleAppender(); + console.setTarget("System.err"); + + String pattern = "%p %C{1} - %m%n"; + console.setLayout(new PatternLayout(pattern)); + console.activateOptions(); + console.setThreshold(Level.DEBUG); + + logger.addAppender(console); + return logger; + } +} diff --git a/erp/com/splunk/erp/core/ERPMain.java b/erp/com/splunk/erp/core/ERPMain.java new file mode 100644 index 00000000..114669c8 --- /dev/null +++ b/erp/com/splunk/erp/core/ERPMain.java @@ -0,0 +1,112 @@ +package com.splunk.erp.core; + +import org.apache.log4j.Logger; +import org.codehaus.jackson.JsonNode; + +import com.splunk.erp.commons.ERPUtils; +import com.splunk.util.WildcardList; + +/** + * This class defines the method for initiating the search from ERP. + * Script erp_script.sh executes the java process with {@link ERPMain} + * as the main class. + * @author smetkar + */ + +public class ERPMain { + + private static Logger logger = ERPUtils.getLogger(ERPMain.class); + +// public static void main(String[] args) throws Exception +// { +// ResultWriter resultWriter = new ResultWriter(ERPUtils.getHostName()); +// +// try{ +// JsonNode argsForERP = ERPUtils.createJsonNodeFromStream(System.in); +// Provider provider = Provider.getProviderInstance(argsForERP); +// +// SearchInfo searchInfo = SearchInfo.getSearchInfoInstance(argsForERP); +// WildcardList requiredFieldList = ERPUtils.getRequiredFieldList(argsForERP); +// +// FieldAppender resultWriterProxy = resultWriter; +// +// provider.init(searchInfo,requiredFieldList); +// +// String record = ""; +// while((record = provider.getNextRecord(resultWriterProxy)) != null){ +// resultWriter.append(record); +// } +// +// provider.close(); +// } catch (Exception ex) { +// ERPLogger.logError("Error while executing ERPMain process - " + ex.getMessage()); +// } finally { +// resultWriter.close(); +// } +// } + + /** + * Splunkd process invokes this method to initiate the search process for ERP. + * + * 1. It is responsible for creating instances of {@link ProviderConfig}, {@link VixConfig}, + * {@link SearchInfo} and {@link WildcardList} required for further processing of search request. + * 2. It controls the flow of search from ERP. + * + * The name of the {@link IProvider} implementation class is passed as an argument while instantiating 'ERPMain' process. + * @param + * @throws Exception + */ + public static void main(String[] args) throws Exception + { + String className = args[0]; + ResultWriter resultWriter = null; + logger.info("Starting search process over external resource provider..."); + try{ + /* S-2-ERP protocol JSON is passed to ERPMain process using System console + * Create JsonNode object from JSON string and create instances of ProviderConfig, VixConfig, + * SearchInfo and WildcardList + */ + logger.info("Reading arguments for search process"); + JsonNode argsForERP = ERPUtils.readArgsForERP(System.in); + logger.info("Creating provider instance with class name : " + className); + IProvider provider = ERPUtils.getProviderInstance(className); + + JsonNode providerConfigNode = ERPUtils.getProviderConfigNode(argsForERP); + JsonNode vixesConfigNode = ERPUtils.getVixesConfigNode(argsForERP); + + logger.info("Obtained provider config"); + ProviderConfig providerConf = ProviderConfig.getProviderConfigInstance(providerConfigNode); + + VixConfig[] vixesConf = VixConfig.getVixesConfig(vixesConfigNode); + logger.info("Obtained virtual indexes config"); + SearchInfo searchInfo = SearchInfo.getSearchInfoInstance(argsForERP); + logger.info("Obtained search info"); + WildcardList requiredFieldList = ERPUtils.getRequiredFieldList(argsForERP); + logger.info("Obtained required field list"); + + logger.info("Creating result writer for provider"); + resultWriter = new ResultWriter(ERPUtils.getHostName()); + + //Pass down the proxy for ResultWriter to IProvider implementing class + logger.info("Creating proxy to result writer"); + ResultWriterProxy resultWriterProxy = resultWriter; + + logger.info("Initializing provider..."); + //Make the ERP ready for fetching results + provider.init(providerConf, vixesConf, searchInfo, requiredFieldList); + //Start fetching the results/records + logger.info("Start record fetching"); + provider.run(resultWriterProxy); + //Close the ERP when results/record fetching is completed. + logger.info("Record fetching complete, closing provider"); + provider.close(); + + } catch (Exception ex) { + logger.error("Error executing search process - " + ex.getMessage(),ex); + } finally { + logger.info("Closing result writer"); + if(resultWriter != null) + resultWriter.close(); + } + } +} diff --git a/erp/com/splunk/erp/core/IProvider.java b/erp/com/splunk/erp/core/IProvider.java new file mode 100644 index 00000000..b5e20798 --- /dev/null +++ b/erp/com/splunk/erp/core/IProvider.java @@ -0,0 +1,52 @@ +package com.splunk.erp.core; + +import java.io.Closeable; + +import com.splunk.util.WildcardList; + +/** + * Interface that an External Resource Provider(ERP) developer needs to implement + * in order to analyse the data stored in external resource (eg. NoSQL database + * like MongoDB, Cassandra etc.) using Hunk. + * @author smetkar + * + */ +public interface IProvider extends Closeable{ + + /** + * An External Resource Provider (ERP) developer implementing this method is expected to + * implement following steps: + * + * 1. Responsible for handling the request from a Splunk/Hunk search, parsing the arguments + * and setting up any necessary resource (connections, files etc). + * 2. Transform the SPL query into a query which is specific to the database/datastore. + * 3. Make use of {@link SearchInfo} and {@link WildcardList} while building query + * + * The method calling init method assumes that the ERP is ready for returning results after calling this method. + * + * @param {@link ProviderConfig} Provider specific properties + * @param {@link VixConfig} Virtual index specific properties + * @param {@link SearchInfo} Search specific information + * @param {@link WildcardList} Required field list as WildcardList object + * @throws Exception : Any exception raised during initialization + */ + public void init(ProviderConfig providerConf, VixConfig[] vixesConf, + SearchInfo searchInfo, WildcardList requiredFieldList) + throws Exception; + + /** + * An External Resource Provider (ERP) implementing this method is expected to + * implement following steps: + * + * 1. If SPL query consist of multiple virtual indexes, ERP developers should handle the fetching + * of results from each virtual index. + * 2. Initialize result writer object of ERP framework by setting index, source and sourcetype and + * other header fields using proxy. + * 2. Handle the batching of results/records and pushing the batch of records/events as List to the proxy. + * 3. Filter the white listed fields, specified using {@link WildcardList}, post querying database/datastore. + * + * @param {@link ResultWriterProxy} Proxy to ResultWriter object. + * @throws Exception : Any exception raised while querying ERP and fetching results. + */ + public void run(ResultWriterProxy resultWriterProxy) throws Exception; +} diff --git a/erp/com/splunk/erp/core/Provider.java b/erp/com/splunk/erp/core/Provider.java new file mode 100644 index 00000000..1b9c60ef --- /dev/null +++ b/erp/com/splunk/erp/core/Provider.java @@ -0,0 +1,98 @@ +package com.splunk.erp.core; + +import java.io.Closeable; +import java.lang.reflect.Constructor; + +import org.codehaus.jackson.JsonNode; + +import com.splunk.erp.commons.ERPLogger; +import com.splunk.erp.commons.ERPUtils; +import com.splunk.util.WildcardList; + +//TODO Keeping this class as we have not decided whether Provider should be an interface or an abstract class + +public abstract class Provider implements Closeable{ + + private ProviderConfig providerConfig; + private VixConfig[] vixesConfig; + + public Provider(ProviderConfig providerConf, VixConfig[] vixesConf) { + this.providerConfig = providerConf; + this.vixesConfig = vixesConf; + } + + public ProviderConfig getProviderConfig() { + return providerConfig; + } + + public VixConfig[] getVixesConfig() { + return vixesConfig; + } + + protected void setProviderConfig(ProviderConfig providerConf) { + this.providerConfig = providerConf; + } + + protected void setVixesConfig(VixConfig[] vixesConf) { + this.vixesConfig = vixesConf; + } + + public abstract void init(SearchInfo searchInfo, WildcardList requiredFieldList) throws Exception; + public abstract String getNextRecord(FieldAppender fieldAppenderImpl) throws Exception; + + public static Provider getProviderInstance(JsonNode argsForERP) { + try + { + JsonNode providerConfigNode = ERPUtils.getProviderConfigNode(argsForERP); + JsonNode vixesConfigNode = ERPUtils.getVixesConfigNode(argsForERP); + + ProviderConfig providerConf = ProviderConfig.getProviderConfigInstance(providerConfigNode); + VixConfig[] vixesConf = VixConfig.getVixesConfig(vixesConfigNode); + + String className = providerConf.getProviderImplClassName(); + Class providerClass = (Class)Class.forName(className); + Constructor constructor = providerClass.getConstructor(new Class[] {ProviderConfig.class, VixConfig.class}); + Provider provider = constructor.newInstance(new Object[] {providerConf,vixesConf}); + return provider; + + } catch (Exception ex) { + ERPLogger.logError("Error while instantiating Provider implementation class, Message :" + ex.getMessage()); + throw new RuntimeException("Error while instantiating Provider implementation class, Message :" + ex.getMessage()); + } + } + + public void run() throws Exception + { + ResultWriter resultWriter = new ResultWriter(ERPUtils.getHostName()); + try{ + JsonNode argsForERP = ERPUtils.readArgsForERP(System.in); + JsonNode providerConfigNode = ERPUtils.getProviderConfigNode(argsForERP); + JsonNode vixesConfigNode = ERPUtils.getVixesConfigNode(argsForERP); + + ProviderConfig providerConf = ProviderConfig.getProviderConfigInstance(providerConfigNode); + VixConfig[] vixesConf = VixConfig.getVixesConfig(vixesConfigNode); + + this.setProviderConfig(providerConf); + this.setVixesConfig(vixesConf); + + SearchInfo searchInfo = SearchInfo.getSearchInfoInstance(argsForERP); + WildcardList requiredFieldList = ERPUtils.getRequiredFieldList(argsForERP); + + ResultWriterProxy resultWriterProxy = resultWriter; + + this.init(searchInfo,requiredFieldList); + + String result = null; + while((result = this.getNextRecord(resultWriterProxy)) != null){ + resultWriter.append(result); + } + + this.close(); + + } catch (Exception ex) { + ERPLogger.logError("Error while executing ERPMain process - " + ex.getMessage()); + } finally { + resultWriter.close(); + } + } +} diff --git a/erp/com/splunk/erp/core/ProviderConfig.java b/erp/com/splunk/erp/core/ProviderConfig.java new file mode 100644 index 00000000..527f6344 --- /dev/null +++ b/erp/com/splunk/erp/core/ProviderConfig.java @@ -0,0 +1,82 @@ +package com.splunk.erp.core; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.codehaus.jackson.JsonNode; + +import com.splunk.erp.commons.ERPUtils; + +/** + * A POJO to represent Provider specific properties. + * The properties/parameters which you define under [provider-family] and [provider] section in indexes.conf + * are combined and passes down in S-2-ERP protocol JSON as 'provider' element under 'conf' JSON element. + * + * @author smetkar + */ + +public class ProviderConfig { + + protected String familyName; + protected String name; + protected String mode; + //Provider specific properties are stored in a Map + protected Map configParams; + + public ProviderConfig(String familyName,String providerName,String mode, Map configParams) { + this.familyName = familyName; + this.configParams = configParams; + this.mode = mode; + this.name = providerName; + } + + public Map getConfigParams() { + return Collections.unmodifiableMap(configParams); + } + + public String getFamilyName() { + return familyName; + } + + public String getProviderName() { + return name; + } + + public String getMode() { + return mode; + } + +// /** +// * Get Provider implementation class name from S-2-ERP protocol JSON +// * @return String +// * @throws IllegalArgumentException +// */ +// public String getProviderImplClassName() throws IllegalArgumentException +// { +// String className = properties.get("class.name"); +// if(className == null) +// throw new IllegalArgumentException("Unable to find the implementing class name"); +// return className; +// } + + /** + * Create ProviderConfig object from S-2-ERP protocol JSON + * @param JsonNode : JSON element for provider specific properties + * @return ProviderConfig object + */ + public static ProviderConfig getProviderConfigInstance(JsonNode providerConfigNode) { + String familyName = ERPUtils.getFamilyName(providerConfigNode); + String providerName = ERPUtils.getProviderName(providerConfigNode); + String mode = ERPUtils.getSearchMode(providerConfigNode); + + Map params = new HashMap(); + Iterator fieldIterator = providerConfigNode.getFieldNames(); + while(fieldIterator.hasNext()){ + String field = fieldIterator.next(); + params.put(field,providerConfigNode.get(field).getTextValue()); + } + return new ProviderConfig(familyName,providerName,mode,params); + } +} diff --git a/erp/com/splunk/erp/core/ResultWriter.java b/erp/com/splunk/erp/core/ResultWriter.java new file mode 100644 index 00000000..1cfdfea2 --- /dev/null +++ b/erp/com/splunk/erp/core/ResultWriter.java @@ -0,0 +1,283 @@ +package com.splunk.erp.core; + +import java.io.ByteArrayOutputStream; +import java.io.FileDescriptor; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.map.JsonMappingException; +import org.codehaus.jackson.map.ObjectMapper; + +import com.splunk.erp.commons.ERPUtils; +import com.splunk.io.ChunkedOutputStream; +import com.splunk.io.SearchOutputStream; + +/** + * This class has methods by which ERP developers push the list of records/documents to Splunkd process.
+ * It takes control of batching the results and streaming the results to Splunkd process.
+ * It implements {@link ResultWriterProxy} interface which defines method to push results, add header fields
+ * and send metrics and messages for search process using {@link SearchOutputStream}. + * @author smetkar + * + */ +public class ResultWriter implements ResultWriterProxy{ + + private static Logger logger = ERPUtils.getLogger(ResultWriter.class); + + private static int OUTPUT_BUFFER_LIMIT = 32 * 1024; + private static int MAX_BUFFER_FLUSH_SIZE = (int) (OUTPUT_BUFFER_LIMIT * 0.9); + private static int INITIAL_BUFFER_FLUSH_SIZE = 1024; + private static String NEWLINE = System.getProperty("line.separator"); + private static int MAX_BATCH_SIZE = 5000; + private static int INITIAL_BATCH_SIZE = 10; + + private boolean isHeaderUpdated; + private ByteArrayOutputStream eventsCollector; + private int bufferFlushSize; + private Map headerFields; + private SearchOutputStream outputStream; + private int count; + private int currentBatchSize; + + + public ResultWriter(String hostName) { + currentBatchSize = INITIAL_BATCH_SIZE; + + //Some default header fields but can be updated using initializeWriter method + headerFields.put("field.host", hostName); + headerFields.put("props.KV_MODE", "json"); + headerFields.put("props.TRUNCATE", "1000000"); + isHeaderUpdated = true; + + eventsCollector = new ByteArrayOutputStream(OUTPUT_BUFFER_LIMIT); + bufferFlushSize = INITIAL_BUFFER_FLUSH_SIZE; + + OutputStream stdout = new FileOutputStream(FileDescriptor.out); + outputStream = new SearchOutputStream(new ChunkedOutputStream(stdout)); + outputStream.setStreamType("raw"); + + System.setOut(System.err); + } + + /** + * Writes content of {@link ByteArrayOutputStream} to {@link SearchOutputStream} and stream header + * if header fields are updated. + * @throws IOException + */ + private void stream() throws IOException{ + if(isHeaderUpdated) { + logger.info("Updating stream header"); + logger.debug("Header fields : " + headerFields); + outputStream.addHeader(headerFields); + isHeaderUpdated = false; + } + outputStream.write(eventsCollector); + eventsCollector.reset(); + } + + /** + * Close SearchOutputStream + * @throws IOException + */ + public void close() { + try + { + if(eventsCollector.size() > 0) + { + stream(); + outputStream.flush(); + } + } catch(IOException ioe) { + logger.error("Error while closing result writer , Message - " + ioe.getMessage()); + } + } + + /** + * Get current batch size + * @return int : current batch size + */ + public int getCurrentBatchSize() { + return currentBatchSize; + } + + @Override + public void initializeWriter(String index, String source, String sourcetype, Map streamHeaderFields) { + logger.info("Initializing result writer"); + headerFields.clear(); + headerFields.put("field.index", index); + headerFields.put("field.source", source); + headerFields.put("field.sourcetype", sourcetype); + headerFields.putAll(streamHeaderFields); + isHeaderUpdated = true; + } + + @Override + public void append(List recordBatch) throws JsonGenerationException,JsonMappingException,IOException { + logger.debug("Appending " + recordBatch.size() + " records"); + //Using third-party library for serializing record/event object to JSON string + ObjectMapper mapper = new ObjectMapper(); + for(Object record : recordBatch) + { + append(mapper.writeValueAsString(record)); + } + } + + @Override + public void append(String result) throws IOException { + eventsCollector.write(result.getBytes()); + eventsCollector.write(NEWLINE.getBytes()); + + //Intermittent streaming of results to give a view of continuous result streaming to user + //Double the buffer flush size for a maximum of MAX_BUFFER_FLUSH_SIZE + if(eventsCollector.size() > bufferFlushSize) + { + stream(); + bufferFlushSize *= 2; + if(bufferFlushSize > MAX_BUFFER_FLUSH_SIZE) + bufferFlushSize = MAX_BUFFER_FLUSH_SIZE; + } + + //keep track of records/events streamed so far + //batchsize is used by ERP developer to understand what is the expected batch size while + //pushing records/events to Splunkd process + count++; + if(count == currentBatchSize) { + currentBatchSize *= 2; + if(currentBatchSize > MAX_BATCH_SIZE) + currentBatchSize = MAX_BATCH_SIZE; + count = 0; + logger.debug("Updating batchsize to " + currentBatchSize); + } + } + +// /** +// * Add new header field +// * Returns true if a new field that is added else false. +// */ +// public boolean addHeaderField(String field, Object fieldValue) { +// boolean isNewField = false; +// +// if(headerFields.get(field) == null){ +// headerFields.put(field,fieldValue); +// isNewField = true; +// this.isHeaderUpdated = true; +// ERPLogger.logInfo("New header field "+ field+ " added with value as " + fieldValue); +// } +// return isNewField; +// } +// +// /** +// * Adding new header fields or updating header fields +// * @param field +// * @param new value for the field +// */ +// public boolean updateHeaderField(String field, Object newVal) +// { +// boolean isFieldUpdated = false; +// Object prevVal = headerFields.get(field); +// +// if(prevVal == null || (newVal != null && !newVal.toString().isEmpty() && !newVal.equals(prevVal))){ +// headerFields.put(field,newVal); +// isFieldUpdated = true; +// isHeaderUpdated = true; +// ERPLogger.logInfo("Header field " + field + " updated with value : "+ newVal); +// } +// return isFieldUpdated; +// } +// +// /** +// * Get value for specific stream header field. Returns field value if present else null. +// * @param key +// * @return value for field +// * +// */ +// public String getHeaderFieldValue(String field) +// { +// if(headerFields.get(field) != null) { +// return headerFields.get(field).toString(); +// } +// return null; +// } +// +// /** +// * Check if a specific header field is present +// */ +// public boolean isHeaderFieldPresent(String field) +// { +// boolean isPresent = false; +// if(headerFields.get(field)!= null) +// isPresent = true; +// return isPresent; +// } + + @Override + public void setTimestampFieldPrefix(String timestampPrefix) { + logger.info("Setting timestamp prefix to " + timestampPrefix); + headerFields.put("props.TIME_PREFIX", timestampPrefix); + isHeaderUpdated = true; + } + + @Override + public void setTimestampFormat(String format) { + logger.info("Setting timestamp format to " + format); + headerFields.put("props.TIME_FORMAT", format); + isHeaderUpdated = true; + } + + /** + * Set count metric for search process + */ + public void addCountMetric(String name, long input, long output) { + logger.info("Setting count metric : " + name + " { input : " + input + " output : " + output +"}"); + outputStream.addCountMetric(name, input, output); + isHeaderUpdated = true; + } + + public void addLink(String name, String url) { + logger.info("Adding link : " + name + " url : " + url); + outputStream.addLink(name, url); + isHeaderUpdated = true; + } + + /** + * Set message for search process + */ + public void addMessage(String message, String value) { + logger.info("Adding message - " + value ); + outputStream.addMessage(message, value); + isHeaderUpdated = true; + } + + /** + * Set exception message for search process + */ + public void addMessage(String message, Exception exception) { + logger.info("Adding exception message - " + message); + outputStream.addMessage(message, exception); + isHeaderUpdated = true; + + } + + /** + * Set metric for search process + */ + public void addMetric(String metricName, long elapsed_ms, long calls) { + logger.info("Adding metric - " + metricName); + outputStream.addMetric(metricName, elapsed_ms, calls); + isHeaderUpdated = true; + } + + /** + * Set prefix for field from metric should be extracted + */ + public void setMetricPrefix(String prefix) { + logger.info("Setting metric prefix to " + prefix); + outputStream.setMetricPrefix(prefix); + isHeaderUpdated = true; + } +} diff --git a/erp/com/splunk/erp/core/ResultWriterProxy.java b/erp/com/splunk/erp/core/ResultWriterProxy.java new file mode 100644 index 00000000..240cbc87 --- /dev/null +++ b/erp/com/splunk/erp/core/ResultWriterProxy.java @@ -0,0 +1,67 @@ +package com.splunk.erp.core; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import org.codehaus.jackson.JsonGenerationException; +import org.codehaus.jackson.map.JsonMappingException; + +import com.splunk.io.SearchMetricsReporter; +import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream; + +/** + * A proxy to append records, header fields and search metrics while streaming search results. + * + * @author smetkar + */ + +public interface ResultWriterProxy extends SearchMetricsReporter { + + /** + * Initialize {@link ResultWriter} object with index name, source and sourcetype of + * virtual index along with other header fields + * @param String index : Virtual index name + * @param String source : Source of search results (eg. database/table) + * @param String sourcetype : Sourcetype of search results + * @param Map of header fields : You can specify values for parameters in props.conf, transform.conf etc. + */ + public void initializeWriter(String index, String source, String sourcetype, Map headerFields) throws Exception; + + + /** + * Method for ERP developers to push results to {@link ResultWriter}. + * Serializes records/documents as JSON string + * @param List of records/documents + * @throws Exception : Throws exception if any serializing object + */ + public void append(List recordBatch) throws JsonGenerationException,JsonMappingException,IOException; + + /** + * This method takes serialized JSON string and keep appending it to {@link ByteOutputStream}. and streams + * when buffer flush size is reached. Controls batchsize and buffer flush size increment. + * @param String : serialied JSON String + * @throws Exception : Throws exception if any while streaming results + */ + public void append(String record) throws IOException; + +// public boolean addHeaderField(String field, String fieldValue); +// +// public String getHeaderFieldValue(String field); +// +// public boolean isHeaderFieldPresent(String field); +// +// public boolean updateHeaderField(String field, String newFieldValue); + + /** + * Set regex of field from which timestamp needs to be extracted + * @param field prefix regex (String) + */ + public void setTimestampFieldPrefix(String timestampField); + + /** + * Set expected timestamp format + * @param timestamp format regex (String) + */ + public void setTimestampFormat(String timestampFormat); +} diff --git a/erp/com/splunk/erp/core/SearchCompareElement.java b/erp/com/splunk/erp/core/SearchCompareElement.java new file mode 100644 index 00000000..3a819c87 --- /dev/null +++ b/erp/com/splunk/erp/core/SearchCompareElement.java @@ -0,0 +1,120 @@ +package com.splunk.erp.core; + +import java.util.HashSet; + +import org.codehaus.jackson.JsonNode; + +/** + * 'search_expr' element in S-2-ERP protocol JSON represents the parsed SPL query. + * POJO to represent search_expr of type 'cmp'.
+ * Example of 'cmp' type search_expr + * + * { + * "type":"cmp", + * "op":"=", + * "lhs":"salary", + * "rhs":"138967.*", + * "is_negated":false, + * "is_numeric":false, + * "is_literal_term":false, + * "is_case_sensitive":false, + * "is_cidr_match":false + * } + * is_numeric attribute states if rhs is numeric + * is_case_sensitive attribute states if the search for string is case_sensitive + * is_literal_term attribute states if the search query is phrase query + * + *
+ * Valid operators for SearchCompareElement are '<', '>' , '<=' , '>=' and '!='. + * @author smetkar + * + */ +public class SearchCompareElement extends SearchElement { + + protected static final HashSet VALID_OP = new HashSet(); + + static{ + VALID_OP.add("="); + VALID_OP.add("!="); + VALID_OP.add("<"); + VALID_OP.add("<="); + VALID_OP.add(">"); + VALID_OP.add(">="); + } + + private String lhs; + private Object rhs; + private boolean is_negated; + private boolean is_numeric; + private boolean is_literal_term; + private boolean is_case_sensitive; + private boolean is_cidr_match; + + public SearchCompareElement(){} + + public SearchCompareElement(String type, String operator, String lhs, Object rhs, boolean is_negated, + boolean is_numeric, boolean is_literal_term, + boolean is_case_sensitive, boolean is_cidr_match) { + super(type, operator); + this.lhs = lhs; + this.rhs = rhs; + this.is_negated = is_negated; + this.is_case_sensitive = is_case_sensitive; + this.is_literal_term = is_literal_term; + this.is_numeric = is_numeric; + } + + /** + * Initialize SearchCompareElement from JsonNode object + */ + public void initFrom(JsonNode compareNode) { + + if(!VALID_OP.contains(compareNode.get("op").getTextValue())) + throw new IllegalArgumentException("Not a valid operator for SearchCompareElement"); + + super.initFrom(compareNode); + lhs = compareNode.get("lhs").getTextValue(); + rhs = compareNode.get("rhs").getTextValue(); + is_numeric = compareNode.get("is_numeric").getBooleanValue(); + is_literal_term = compareNode.get("is_literal_term").getBooleanValue(); + is_case_sensitive = compareNode.get("is_case_sensitive").getBooleanValue(); + is_cidr_match = compareNode.get("is_cidr_match").getBooleanValue(); + + /* + * If rhs is numeric convert it into Double object else check if it is not case-sensitive and lowercase rhs. + */ + if(is_numeric) { + rhs = Double.parseDouble(rhs.toString()); + } else if(!is_case_sensitive) { + rhs = rhs.toString().toLowerCase(); + } + } + + public String getLhs() { + return lhs; + } + + public Object getRhs() { + return rhs; + } + + public boolean is_negated() { + return is_negated; + } + + public boolean is_numeric() { + return is_numeric; + } + + public boolean is_literal_term() { + return is_literal_term; + } + + public boolean is_case_sensitive() { + return is_case_sensitive; + } + + public boolean is_cidr_match() { + return is_cidr_match; + } +} diff --git a/erp/com/splunk/erp/core/SearchElement.java b/erp/com/splunk/erp/core/SearchElement.java new file mode 100644 index 00000000..50a72788 --- /dev/null +++ b/erp/com/splunk/erp/core/SearchElement.java @@ -0,0 +1,43 @@ +package com.splunk.erp.core; + +import org.codehaus.jackson.JsonNode; + +public class SearchElement { + + protected String type; + protected String operator; + + public SearchElement(){ + } + + public SearchElement(String type, String operator) { + this.operator = operator; + this.type = type; + } + + public String getOperator() { + return operator; + } + + public String getType() { + return type; + } + + public void initFrom(JsonNode node) { + this.operator = node.get("op").getTextValue(); + this.type = node.get("type").getTextValue(); + } + + public static SearchElement getByType(String type) throws IllegalArgumentException{ + if(type.equalsIgnoreCase("cmp")) + return new SearchCompareElement(); + else if (type.equalsIgnoreCase("group")) + return new SearchGroupElement(); + //TODO Implement sublclasses of SearchElement for type 'term' and 'phrase' + else if(type.equalsIgnoreCase("term")) + return null; + else if(type.equalsIgnoreCase("phrase")) + return null; + throw new IllegalArgumentException("Unknown node type = "+type); + } +} diff --git a/erp/com/splunk/erp/core/SearchGroupElement.java b/erp/com/splunk/erp/core/SearchGroupElement.java new file mode 100644 index 00000000..e382ac5c --- /dev/null +++ b/erp/com/splunk/erp/core/SearchGroupElement.java @@ -0,0 +1,82 @@ +package com.splunk.erp.core; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; + +import org.codehaus.jackson.JsonNode; + +/** + * 'search_expr' element in S-2-ERP protocol JSON represents the parsed SPL query. + * POJO to represent 'search_expr' of type 'group' + * Example of 'group' type search_expr + * { type":"group", + * "op":"AND", + * "children": + * [ + * { + * "type":"cmp", + * "op":"=", + * "lhs":"salary", + * "rhs":"138967.*", + * "is_negated":false, + * "is_numeric":false, + * "is_literal_term":false, + * "is_case_sensitive":false, + * "is_cidr_match":false + * } + * ] + * } + * Valid operators are AND , OR + * A parsed SPL query can contain multiple 'AND'/'OR' condition and children attribute represents the different search condition. + * They can be either of type 'cmp' or type 'group'. + * + * @author smetkar + */ + +public class SearchGroupElement extends SearchElement{ + + public static HashSet VALID_OP = new HashSet(); + + static { + VALID_OP.add("AND"); + VALID_OP.add("OR"); + } + + protected ArrayList children; + + public SearchGroupElement() {} + + public SearchGroupElement(String type,String operator, ArrayList children) { + super(type,operator); + this.children = children; + } + + public ArrayList getChildren() { + return children; + } + + /** + * Initialize SearchGroupElement from JsonNode arguments + * @param JsonNode object + */ + public void initFrom(JsonNode groupNode) throws IllegalArgumentException { + + if(!VALID_OP.contains(groupNode.get("op").getTextValue())) + throw new IllegalArgumentException("Not a valid operator for SearchGroupElement"); + + super.initFrom(groupNode); + JsonNode childrenNode = groupNode.get("children"); + if(childrenNode.isArray()) { + Iterator childIterator = childrenNode.getElements(); + this.children = new ArrayList(); + while(childIterator.hasNext()) + { + JsonNode childNode = childIterator.next(); + SearchElement child = SearchElement.getByType(childNode.get("type").getTextValue()); + child.initFrom(childNode); + this.children.add(child); + } + } + } +} diff --git a/erp/com/splunk/erp/core/SearchInfo.java b/erp/com/splunk/erp/core/SearchInfo.java new file mode 100644 index 00000000..1a88055c --- /dev/null +++ b/erp/com/splunk/erp/core/SearchInfo.java @@ -0,0 +1,136 @@ +package com.splunk.erp.core; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import org.codehaus.jackson.JsonNode; + +/** + * POJO to represent 'info' element of S-2-ERP protocol JSON found under 'search' element. + * It is a list of key-value pair and stored into Map. + * + * @author smetkar + */ +public class SearchInfo { + + private Map fields; + + public SearchInfo(Map fields) { + this.fields = fields; + } + + /** + * Get value for particular field + * @param fieldName + * @return + */ + public String getFieldValue(String fieldName) { + return fields.get(fieldName); + } + + /** + * Get all set of fields available in info element of JSON + * @return + */ + public Set getSearchInfoFieldSet() { + return fields.keySet(); + } + + /** + * Check if specified field is present + * @param fieldName + * @return boolean + */ + public boolean hasField(String fieldName) { + return fields.get(fieldName) != null ? true : false; + } + + /** + * Get value for _timestamp field from info element of JSON + * @return Double (timestamp) + */ + public Double getTimestampInfo() { + String timestamp = fields.get("_timestamp"); + Double timestampVal = null; + if(timestamp != null) { + timestampVal = Double.parseDouble(timestamp); + } + return timestampVal; + } + + /** + * Check if it is a Time range query + * It is a time range query if the info element of S-2-ERP protocol JSON contains '_search_et' or '_search_lt' element + * @return + */ + public boolean isTimeRangeQuery() { + return (fields.get("_search_et") != null || fields.get("_search_lt") != null) ? true : false; + } + + /** + * Get earliest time for search + * @return Double (timestamp) + */ + public Double getEarliestTime() { + String timestamp = fields.get("_search_et"); + Double timestampVal = null; + if(timestamp != null) { + timestampVal = Double.parseDouble(timestamp); + } + return timestampVal; + } + + /** + * Get latest time for search + * @return Double (timestamp) + */ + public Double getLatestTime() { + String timestamp = fields.get("_search_lt"); + Double timestampVal = null; + if(timestamp != null) { + timestampVal = Double.parseDouble(timestamp); + } + return timestampVal; + } + + /** + * Get user session authentication token + * + * @return String : user session token + */ + public String getAuthenticationToken() { + return fields.get("_auth_token"); + } + + /** + * Creates a SearchInfo object from S-2-ERP protocol JSON + * @param JsonNode : JSON element containing search info + * @return SearchInfo object + */ + public static SearchInfo getSearchInfoInstance(JsonNode searchInfoNode) { + SearchInfo searchInfo = null; + JsonNode infoNode = searchInfoNode.get("args").get("search").get("info"); + if(infoNode != null) { + Map fields = new HashMap(); + if(infoNode.isArray()) { + Iterator it = infoNode.getElements(); + while(it.hasNext()){ + JsonNode node = it.next(); + Iterator fieldNames = node.getFieldNames(); + while(fieldNames.hasNext()){ + String fieldName = fieldNames.next(); + //Ignoring '_tz' property because of unnecessary data present in it + if(!fieldName.equalsIgnoreCase("_tz")) + fields.put(fieldName,node.get(fieldName).getTextValue()); + } + } + if(fields != null) { + searchInfo = new SearchInfo(fields); + } + } + } + return searchInfo; + } +} diff --git a/erp/com/splunk/erp/core/VixConfig.java b/erp/com/splunk/erp/core/VixConfig.java new file mode 100644 index 00000000..c136a933 --- /dev/null +++ b/erp/com/splunk/erp/core/VixConfig.java @@ -0,0 +1,82 @@ +package com.splunk.erp.core; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +import org.codehaus.jackson.JsonNode; + +/** + * POJO to represent virtual index specific properties. + * Virtual index specific parameters are stored in Map + * The 'search_expr' element of S-2-ERP protocol JSON is reprsented by {@link SearchElement} object. + * + * @author smetkar + */ +public class VixConfig { + + private String name; + private Map configParams; + private String provider; + //TODO Decide on whether we want to use subclass of ParseNode or subclass of SearchElement + private SearchElement searchExpression; + + public VixConfig(String indexName,String provider, Map configParams,SearchElement searchExpression) { + this.name = indexName; + this.provider = provider; + this.configParams = configParams; + this.searchExpression = searchExpression; + } + + public SearchElement getSearchExpression() { + return searchExpression; + } + + public Map getConfigParams() { + return Collections.unmodifiableMap(configParams); + } + + public String getIndexName() { + return name; + } + + public String getProviderName() { + return provider; + } + + /** + * Create virtual indexes configs from S-2-ERP protocol JSON + * @param familyName(Provider family name) + * @param indexNode(Json node containing index information) + * @return VixConfig array + */ + public static VixConfig[] getVixesConfig(JsonNode vixesConfigNode) + { + VixConfig[] vixesConfig = null; + Map params = null; + + vixesConfig = new VixConfig[vixesConfigNode.size()]; + Iterator it = vixesConfigNode.getElements(); + int vixCount = 0; + while (it.hasNext()) { + JsonNode vixConfigNode = it.next(); + params = new HashMap(); + Iterator fieldIterator = vixConfigNode.getFieldNames(); + + while(fieldIterator.hasNext()){ + String field = fieldIterator.next(); + params.put(field,vixConfigNode.get(field).getTextValue()); + } + + //Represent parsed SPL query as SearchElement + JsonNode searchExprNode = vixConfigNode.get("search_expr"); + SearchElement searchExpression = SearchElement.getByType(searchExprNode.get("type").getTextValue()); + searchExpression.initFrom(searchExprNode); + String vixName = vixConfigNode.get("name").getTextValue(); + String providerName = vixConfigNode.get("provider").getTextValue(); + vixesConfig[vixCount++] = new VixConfig(vixName, providerName, params, searchExpression); + } + return vixesConfig; + } +} diff --git a/erp/scripts/create_package.py b/erp/scripts/create_package.py new file mode 100644 index 00000000..ec5211a1 --- /dev/null +++ b/erp/scripts/create_package.py @@ -0,0 +1,174 @@ +import sys +import os +import shutil +import io + +# Script to create directory structure and create config file from template files + +# python create_package.py [app_name] [family_name] [class_name] [app_version] [path_to_folder_with_jars] + +def main(args): + app_name = args[1] + family_name = args[2] + class_name = args[3] + version = args[4] + path_to_folder_with_jars = args[5] + + create_directory_structure(app_name) + + default_folder = os.path.join(os.getcwd(),app_name,"default") + if not os.path.exists(default_folder): + print "Unable to find default directory" + exit(0); + + create_config_files(default_folder,family_name,app_name,class_name,version) + + os.chdir(app_name) + copy_jars_to_bin(path_to_folder_with_jars) + create_bash_script_file(app_name) + +def create_directory_structure(app_name): + + if os.path.exists(app_name): + shutil.rmtree(app_name) + + os.makedirs(app_name) + os.chdir(os.path.join(os.getcwd(),app_name)) + + os.chdir(os.path.join(os.getcwd(),"..")) + + static_folder_path = os.path.join(app_name,"appserver","static") + + if not os.path.exists(static_folder_path): + print "creating appserver/static folder" + os.makedirs(static_folder_path) + create_html_documentation(app_name, static_folder_path) + + + default_folder = app_name + "/default" + + if not os.path.exists(default_folder): + print "Creating default folder" + os.makedirs(default_folder) + + data_folder = default_folder + "/data" + + if not os.path.exists(data_folder): + print "Creating data folder" + os.makedirs(data_folder) + + ui_folder = data_folder + "/ui" + + if not os.path.exists(ui_folder): + print "Creating ui folder" + os.makedirs(ui_folder) + + nav_folder = ui_folder + "/nav" + + if not os.path.exists(nav_folder): + print "Creating nav folder" + os.makedirs(nav_folder) + create_default_file(nav_folder) + + views_folder = ui_folder + "/views" + + if not os.path.exists(views_folder): + print "Creating views folder" + os.makedirs(views_folder) + create_documentation_file(app_name, views_folder) + +def copy_jars_to_bin(path_to_folder_with_jars): + print "Path to jars : " + path_to_folder_with_jars + try: + shutil.copytree(path_to_folder_with_jars,"bin") + except shutil.Error as e: + print('Directory not copied. Error: %s' % e) + except OSError as e: + print('Directory not copied. Error: %s' % e) + +def create_config_files(default_folder_path,family_name,app_name,class_name,version): + print "Creating config files" + create_indexes_conf_file(default_folder_path,family_name,app_name,class_name) + create_app_conf_file(default_folder_path,family_name,version) + +def create_bash_script_file(app_name): + print "Creating erp_script.sh file" + file_name = os.path.join(os.getcwd(),"bin","erp_script.sh") + template_file_name = os.path.abspath(os.getcwd() + "/../../template_files/erp_script.sh") + config = io.open(file_name,'w') + + for line in io.open(template_file_name, 'r'): + line = line.replace('$app-name',app_name) + config.write(line) + + config.close() + +def create_default_file(nav_folder_path): + print "Creating default.xml file" + file_name = os.path.join(nav_folder_path,"default.xml") + template_file_name = os.path.abspath(os.getcwd() + "/../template_files/default.xml") + config = io.open(file_name,'w') + + for line in io.open(template_file_name, 'r'): + config.write(line) + + config.close() + +def create_documentation_file(app_name, views_folder_path): + print "Creating Documentation.xml file" + file_name = os.path.join(views_folder_path, "Documentation.xml") + template_file_name = os.path.abspath(os.getcwd() + "/../template_files/Documentation.xml") + config = io.open(file_name,'w') + + for line in io.open(template_file_name, 'r'): + line = line.replace('$app-name',app_name) + config.write(line) + + config.close() + +def create_indexes_conf_file(default_folder_path,family_name,app_name,class_name): + print "Creating indexes.conf file" + file_name = os.path.join(default_folder_path,"indexes.conf") + template_file_name = os.path.abspath(os.getcwd() + "/../template_files/indexes.conf") + config = io.open(file_name,'w') + + for line in io.open(template_file_name, 'r'): + line = line.replace('$app-name',app_name) + line = line.replace('$family-name',family_name) + line = line.replace('$class-name',class_name) + config.write(line) + + config.close() + +def create_app_conf_file(default_folder_path,family_name,version): + print "Creating app.conf file" + file_name = os.path.join(default_folder_path,"app.conf") + template_file_name = os.path.abspath(os.getcwd() + "/../template_files/app.conf") + config = io.open(file_name,'w') + + for line in io.open(template_file_name, 'r'): + line = line.replace('$family-name',family_name) + line = line.replace('$version',version) + config.write(line) + + config.close() + +def create_html_documentation(app_name, static_folder_path): + print "Creating Documentation web page for App with name " + app_name + ".html" + file_name = os.path.join(static_folder_path,app_name + ".html") + + data = "\n\n" + app_name + "\n\nWeb page to include your app documentation. Please refer sample documentation template file (App_Documentation.html) to understand more about the structure of web page.\n"; + fp = open(file_name,'w') + fp.write(data) + fp.close() + +if __name__ == '__main__': + + args = sys.argv + + if args.__len__() != 6: + print "Please enter correct arguments" + print "Syntax : main_class $[app_name] $[family_name] $[complete_class_name] $[app-version] $[path_to_folder_with_jars]" + exit + else: + main(args) \ No newline at end of file diff --git a/erp/template_files/App_Documentation.html b/erp/template_files/App_Documentation.html new file mode 100644 index 00000000..dc9c4a34 --- /dev/null +++ b/erp/template_files/App_Documentation.html @@ -0,0 +1,92 @@ + + + + + Sample Documentation page for MongoDB App + + + + + + + + + + + + + + + + +
+
+ +
+ + + +

MongoDB App

+ + +
+ + + + + +
+

Contents

+ +
+ +

Overview

+

MongoDB is a popular document-oriented database with dynamic schema and provides efficient way to store and retrieve documents with minimum latency. Using Hunk you can tap into the insights hidden into your data and use it to get better understanding of huge amount of data stored in your MongoDB database. Hunk enables you to explore, analyze and visualize the data and get better usable insights. Using MongoDB App, Hunk users can take advantage of Splunk`s interactive query and analytic dashboard using Hunk platform.

+ +

Specifying MongoDB Provider

+

Once you have uploaded the app using 'Manage App' menu, a new app named 'MongoDB App' will be added in your list of installed apps. You need to specify the required MongoDB host details using 'Edit Provider' feature available under Virtual Indexes which can be found in Settings tab.

+
    +

  • MongoDB Host details

    +

    MongoDB host details can be provided using 'vix.mongodb.host' property. Please note that this property can also be set in 'indexes.conf'. 

     eg. vix.mongodb.host = localhost:27017
    +

    +
    • +
+ +

Specifying Virtual Indexes

+

When you install the 'MongoDB App', you can see a virtual index, named 'mongodb_vix' (spcific to MongoDB) created under Virtual Indexes list. The virtual index helps you to specify the details about the collection and databse name from which you want to analyse the data. +

    +

  • Understanding Virtual Index configuration

    + + + + + + + + + +
    Attribute Explaination 
       vix.mongodb.collection  
    		  Name of the collection from which to analyse the data
    +
    • + +
+

+
+ + diff --git a/erp/template_files/Documentation.xml b/erp/template_files/Documentation.xml new file mode 100644 index 00000000..d506a18c --- /dev/null +++ b/erp/template_files/Documentation.xml @@ -0,0 +1,6 @@ + + +