You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/01/24 00:25:18 UTC
[1/2] incubator-spot git commit: SPOT-249. Update Windows Event Log
ingest to support SDC Edge
Repository: incubator-spot
Updated Branches:
refs/heads/SPOT-181_ODM 85d29d414 -> e62631bb2
SPOT-249. Update Windows Event Log ingest to support SDC Edge
This updates the ODMWindowsEventLogs pipeline to consume off of an HTTP Server origin,
which will receive data from an SDC Edge pipeline sending System, Security, or Application
logs over an HTTP connection. This is intended as a configurable starting point, as there
will likely be additional customization required for any real-world environment. SDC
does not easily run on Windows, so SDC Edge will allow offload of Windows logs to an SDC
pipeline that delivers data into the ODM schema.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/5897dc4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/5897dc4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/5897dc4b
Branch: refs/heads/SPOT-181_ODM
Commit: 5897dc4bc02409bb808e4ff7b433781d6173a39b
Parents: 1e94da9
Author: Jon Natkins <na...@streamsets.com>
Authored: Tue Jan 16 14:22:50 2018 -0800
Committer: Jon Natkins <na...@streamsets.com>
Committed: Tue Jan 16 14:25:44 2018 -0800
----------------------------------------------------------------------
.../streamsets/windows/ODMWindowsEventLogs.json | 305 +++++-----
.../streamsets/windows/WindowsHTTPEdge.json | 603 +++++++++++++++++++
2 files changed, 770 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/5897dc4b/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json b/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
index e5a8eaa..1e8bd39 100644
--- a/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
+++ b/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
@@ -1,11 +1,11 @@
{
"pipelineConfig" : {
- "schemaVersion" : 4,
+ "schemaVersion" : 5,
"version" : 7,
"pipelineId" : "ODMWindowsEventLogs9fae6da4-eb7f-4aca-820a-1f561a40e7cc",
"title" : "ODMWindowsEventLogs",
"description" : "",
- "uuid" : "a155ab31-89d9-44d9-94d3-c06fc1fda992",
+ "uuid" : "3c91ae21-f045-413a-b316-c46298982991",
"configuration" : [ {
"name" : "executionMode",
"value" : "STANDALONE"
@@ -48,8 +48,11 @@
"key" : "HIVE_URL",
"value" : ""
}, {
- "key" : "WINDOWS_EVENT_DIR",
- "value" : ""
+ "key" : "WINDOWS_HTTP_APP_ID",
+ "value" : "windowsSpotApp"
+ }, {
+ "key" : "HTTP_PORT",
+ "value" : "8000"
} ]
}, {
"name" : "badRecordsHandling",
@@ -90,6 +93,9 @@
}, {
"name" : "statsAggregatorStage",
"value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
+ }, {
+ "name" : "shouldCreateFailureSnapshot",
+ "value" : true
} ],
"uiInfo" : {
"previewConfig" : {
@@ -104,254 +110,265 @@
}
},
"stages" : [ {
- "instanceName" : "Directory_01",
+ "instanceName" : "HTTPServer_01",
"library" : "streamsets-datacollector-basic-lib",
- "stageName" : "com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource",
- "stageVersion" : "9",
+ "stageName" : "com_streamsets_pipeline_stage_origin_httpserver_HttpServerDPushSource",
+ "stageVersion" : "10",
"configuration" : [ {
- "name" : "conf.dataFormatConfig.compression",
+ "name" : "httpConfigs.tlsConfigBean.tlsEnabled",
+ "value" : false
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.keyStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.keyStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.keyStorePassword",
+ "value" : null
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.keyStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.useDefaultProtocols",
+ "value" : true
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.protocols",
+ "value" : [ ]
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.useDefaultCiperSuites",
+ "value" : true
+ }, {
+ "name" : "httpConfigs.tlsConfigBean.cipherSuites",
+ "value" : [ ]
+ }, {
+ "name" : "httpConfigs.port",
+ "value" : "${HTTP_PORT}"
+ }, {
+ "name" : "httpConfigs.maxConcurrentRequests",
+ "value" : 10
+ }, {
+ "name" : "httpConfigs.appId",
+ "value" : "${WINDOWS_HTTP_APP_ID}"
+ }, {
+ "name" : "httpConfigs.appIdViaQueryParamAllowed",
+ "value" : false
+ }, {
+ "name" : "maxRequestSizeMB",
+ "value" : 100
+ }, {
+ "name" : "dataFormat",
+ "value" : "SDC_JSON"
+ }, {
+ "name" : "dataFormatConfig.compression",
"value" : "NONE"
}, {
- "name" : "conf.dataFormatConfig.filePatternInArchive",
+ "name" : "dataFormatConfig.filePatternInArchive",
"value" : "*"
}, {
- "name" : "conf.dataFormatConfig.charset",
+ "name" : "dataFormatConfig.charset",
"value" : "UTF-8"
}, {
- "name" : "conf.dataFormatConfig.removeCtrlChars",
+ "name" : "dataFormatConfig.removeCtrlChars",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.textMaxLineLen",
+ "name" : "dataFormatConfig.textMaxLineLen",
"value" : 1024
}, {
- "name" : "conf.dataFormatConfig.useCustomDelimiter",
+ "name" : "dataFormatConfig.useCustomDelimiter",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.customDelimiter",
+ "name" : "dataFormatConfig.customDelimiter",
"value" : "\\r\\n"
}, {
- "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
+ "name" : "dataFormatConfig.includeCustomDelimiterInTheText",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.jsonContent",
+ "name" : "dataFormatConfig.jsonContent",
"value" : "MULTIPLE_OBJECTS"
}, {
- "name" : "conf.dataFormatConfig.jsonMaxObjectLen",
- "value" : 4096000
+ "name" : "dataFormatConfig.jsonMaxObjectLen",
+ "value" : 4096
}, {
- "name" : "conf.dataFormatConfig.csvFileFormat",
+ "name" : "dataFormatConfig.csvFileFormat",
"value" : "CSV"
}, {
- "name" : "conf.dataFormatConfig.csvHeader",
+ "name" : "dataFormatConfig.csvHeader",
"value" : "NO_HEADER"
}, {
- "name" : "conf.dataFormatConfig.csvMaxObjectLen",
- "value" : 1024
- }, {
- "name" : "conf.dataFormatConfig.csvAllowExtraColumns",
+ "name" : "dataFormatConfig.csvAllowExtraColumns",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
+ "name" : "dataFormatConfig.csvExtraColumnPrefix",
"value" : "_extra_"
}, {
- "name" : "conf.dataFormatConfig.csvCustomDelimiter",
+ "name" : "dataFormatConfig.csvMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "dataFormatConfig.csvCustomDelimiter",
"value" : "|"
}, {
- "name" : "conf.dataFormatConfig.csvCustomEscape",
+ "name" : "dataFormatConfig.csvCustomEscape",
"value" : "\\"
}, {
- "name" : "conf.dataFormatConfig.csvCustomQuote",
+ "name" : "dataFormatConfig.csvCustomQuote",
"value" : "\""
}, {
- "name" : "conf.dataFormatConfig.csvEnableComments",
+ "name" : "dataFormatConfig.csvEnableComments",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.csvCommentMarker",
+ "name" : "dataFormatConfig.csvCommentMarker",
"value" : "#"
}, {
- "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
+ "name" : "dataFormatConfig.csvIgnoreEmptyLines",
"value" : true
}, {
- "name" : "conf.dataFormatConfig.csvRecordType",
+ "name" : "dataFormatConfig.csvRecordType",
"value" : "LIST_MAP"
}, {
- "name" : "conf.dataFormatConfig.csvSkipStartLines",
+ "name" : "dataFormatConfig.csvSkipStartLines",
"value" : 0
}, {
- "name" : "conf.dataFormatConfig.parseNull",
+ "name" : "dataFormatConfig.parseNull",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.nullConstant",
+ "name" : "dataFormatConfig.nullConstant",
"value" : "\\\\N"
}, {
- "name" : "conf.dataFormatConfig.xmlRecordElement",
+ "name" : "dataFormatConfig.xmlRecordElement",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
+ "name" : "dataFormatConfig.includeFieldXpathAttributes",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.xPathNamespaceContext",
+ "name" : "dataFormatConfig.xPathNamespaceContext",
"value" : [ ]
}, {
- "name" : "conf.dataFormatConfig.outputFieldAttributes",
+ "name" : "dataFormatConfig.outputFieldAttributes",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.xmlMaxObjectLen",
+ "name" : "dataFormatConfig.xmlMaxObjectLen",
"value" : 4096
}, {
- "name" : "conf.dataFormatConfig.logMode",
+ "name" : "dataFormatConfig.logMode",
"value" : "COMMON_LOG_FORMAT"
}, {
- "name" : "conf.dataFormatConfig.logMaxObjectLen",
+ "name" : "dataFormatConfig.logMaxObjectLen",
"value" : 1024
}, {
- "name" : "conf.dataFormatConfig.retainOriginalLine",
+ "name" : "dataFormatConfig.retainOriginalLine",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.customLogFormat",
+ "name" : "dataFormatConfig.customLogFormat",
"value" : "%h %l %u %t \"%r\" %>s %b"
}, {
- "name" : "conf.dataFormatConfig.regex",
+ "name" : "dataFormatConfig.regex",
"value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
}, {
- "name" : "conf.dataFormatConfig.fieldPathsToGroupName",
+ "name" : "dataFormatConfig.fieldPathsToGroupName",
"value" : [ {
"fieldPath" : "/",
"group" : 1
} ]
}, {
- "name" : "conf.dataFormatConfig.grokPatternDefinition",
+ "name" : "dataFormatConfig.grokPatternDefinition",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.grokPattern",
+ "name" : "dataFormatConfig.grokPattern",
"value" : "%{COMMONAPACHELOG}"
}, {
- "name" : "conf.dataFormatConfig.onParseError",
+ "name" : "dataFormatConfig.onParseError",
"value" : "ERROR"
}, {
- "name" : "conf.dataFormatConfig.maxStackTraceLines",
+ "name" : "dataFormatConfig.maxStackTraceLines",
"value" : 50
}, {
- "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
+ "name" : "dataFormatConfig.enableLog4jCustomLogFormat",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.log4jCustomLogFormat",
+ "name" : "dataFormatConfig.log4jCustomLogFormat",
"value" : "%r [%t] %-5p %c %x - %m%n"
}, {
- "name" : "conf.dataFormatConfig.avroSchema",
+ "name" : "dataFormatConfig.avroSchema",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.schemaRegistryUrls",
+ "name" : "dataFormatConfig.schemaRegistryUrls",
"value" : [ ]
}, {
- "name" : "conf.dataFormatConfig.schemaLookupMode",
+ "name" : "dataFormatConfig.schemaLookupMode",
"value" : "SUBJECT"
}, {
- "name" : "conf.dataFormatConfig.subject",
+ "name" : "dataFormatConfig.subject",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.schemaId",
+ "name" : "dataFormatConfig.schemaId",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.protoDescriptorFile",
+ "name" : "dataFormatConfig.protoDescriptorFile",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.messageType",
+ "name" : "dataFormatConfig.messageType",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.isDelimited",
+ "name" : "dataFormatConfig.isDelimited",
"value" : true
}, {
- "name" : "conf.dataFormatConfig.binaryMaxObjectLen",
+ "name" : "dataFormatConfig.binaryMaxObjectLen",
"value" : 1024
}, {
- "name" : "conf.dataFormatConfig.datagramMode",
+ "name" : "dataFormatConfig.datagramMode",
"value" : "SYSLOG"
}, {
- "name" : "conf.dataFormatConfig.typesDbPath",
+ "name" : "dataFormatConfig.typesDbPath",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.convertTime",
+ "name" : "dataFormatConfig.convertTime",
"value" : false
}, {
- "name" : "conf.dataFormatConfig.excludeInterval",
+ "name" : "dataFormatConfig.excludeInterval",
"value" : true
}, {
- "name" : "conf.dataFormatConfig.authFilePath",
+ "name" : "dataFormatConfig.authFilePath",
"value" : null
}, {
- "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
- "value" : 8192
- }, {
- "name" : "conf.dataFormatConfig.rateLimit",
- "value" : "-1"
- }, {
- "name" : "conf.dataFormat",
- "value" : "JSON"
- }, {
- "name" : "conf.spoolDir",
- "value" : "${WINDOWS_EVENT_DIR}"
+ "name" : "dataFormatConfig.netflowOutputValuesMode",
+ "value" : "RAW_AND_INTERPRETED"
}, {
- "name" : "conf.pathMatcherMode",
- "value" : "GLOB"
- }, {
- "name" : "conf.filePattern",
- "value" : "*"
- }, {
- "name" : "conf.useLastModified",
- "value" : "TIMESTAMP"
- }, {
- "name" : "conf.processSubdirectories",
- "value" : false
- }, {
- "name" : "conf.allowLateDirectory",
- "value" : false
+ "name" : "dataFormatConfig.maxTemplateCacheSize",
+ "value" : -1
}, {
- "name" : "conf.overrunLimit",
- "value" : 128
+ "name" : "dataFormatConfig.templateCacheTimeoutMs",
+ "value" : -1
}, {
- "name" : "conf.batchSize",
- "value" : 1000
+ "name" : "dataFormatConfig.netflowOutputValuesModeDatagram",
+ "value" : "RAW_AND_INTERPRETED"
}, {
- "name" : "conf.poolingTimeoutSecs",
- "value" : 600
+ "name" : "dataFormatConfig.maxTemplateCacheSizeDatagram",
+ "value" : -1
}, {
- "name" : "conf.maxSpoolFiles",
- "value" : 1000
+ "name" : "dataFormatConfig.templateCacheTimeoutMsDatagram",
+ "value" : -1
}, {
- "name" : "conf.initialFileToProcess",
- "value" : null
- }, {
- "name" : "conf.errorArchiveDir",
- "value" : null
- }, {
- "name" : "conf.postProcessing",
- "value" : "NONE"
- }, {
- "name" : "conf.archiveDir",
- "value" : null
+ "name" : "dataFormatConfig.wholeFileMaxObjectLen",
+ "value" : 8192
}, {
- "name" : "conf.retentionTimeMins",
- "value" : 0
+ "name" : "dataFormatConfig.rateLimit",
+ "value" : "-1"
}, {
"name" : "stageOnRecordError",
"value" : "TO_ERROR"
} ],
"uiInfo" : {
- "yPos" : 50,
- "stageType" : "SOURCE",
- "rawSource" : {
- "configuration" : [ {
- "name" : "fileName"
- } ]
- },
"description" : "",
- "label" : "Read Windows Event Files",
- "xPos" : 60
+ "label" : "HTTP Server 1",
+ "xPos" : 60,
+ "yPos" : 50,
+ "stageType" : "SOURCE"
},
"inputLanes" : [ ],
- "outputLanes" : [ "Directory_01OutputLane15059716464100" ],
- "eventLanes" : [ ]
+ "outputLanes" : [ "HTTPServer_01OutputLane15161394829660" ],
+ "eventLanes" : [ ],
+ "services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_01",
"library" : "streamsets-datacollector-basic-lib",
@@ -413,9 +430,10 @@
"yPos" : 50,
"stageType" : "PROCESSOR"
},
- "inputLanes" : [ "Directory_01OutputLane15059716464100" ],
+ "inputLanes" : [ "HTTPServer_01OutputLane15161394829660" ],
"outputLanes" : [ "ExpressionEvaluator_01OutputLane15059452532370" ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
}, {
"instanceName" : "FieldRemover_01",
"library" : "streamsets-datacollector-basic-lib",
@@ -443,7 +461,8 @@
},
"inputLanes" : [ "ExpressionEvaluator_01OutputLane15059452532370" ],
"outputLanes" : [ "FieldRemover_01OutputLane15059461007910" ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
}, {
"instanceName" : "HadoopFS_01",
"library" : "streamsets-datacollector-cdh_5_11-lib",
@@ -651,7 +670,8 @@
},
"inputLanes" : [ "FieldRemover_01OutputLane15059461007910" ],
"outputLanes" : [ ],
- "eventLanes" : [ "HadoopFS_01_EventLane" ]
+ "eventLanes" : [ "HadoopFS_01_EventLane" ],
+ "services" : [ ]
}, {
"instanceName" : "ExpressionEvaluator_02",
"library" : "streamsets-datacollector-basic-lib",
@@ -696,7 +716,8 @@
},
"inputLanes" : [ "HadoopFS_01_EventLane" ],
"outputLanes" : [ "ExpressionEvaluator_02OutputLane15059713441990" ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
}, {
"instanceName" : "HiveQuery_01",
"library" : "streamsets-datacollector-cdh_5_11-lib",
@@ -729,6 +750,9 @@
}, {
"name" : "stageRecordPreconditions",
"value" : [ ]
+ }, {
+ "name" : "config.hiveConfigBean.driverProperties",
+ "value" : [ ]
} ],
"uiInfo" : {
"description" : "",
@@ -739,7 +763,8 @@
},
"inputLanes" : [ "ExpressionEvaluator_02OutputLane15059713441990" ],
"outputLanes" : [ ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
} ],
"errorStage" : {
"instanceName" : "Discard_ErrorStage",
@@ -756,28 +781,29 @@
},
"inputLanes" : [ ],
"outputLanes" : [ ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
},
"info" : {
"pipelineId" : "ODMWindowsEventLogs9fae6da4-eb7f-4aca-820a-1f561a40e7cc",
"title" : "ODMWindowsEventLogs",
"description" : "",
"created" : 1505971638546,
- "lastModified" : 1505972055258,
+ "lastModified" : 1516141132063,
"creator" : "natty@dpmfield",
"lastModifier" : "natty@dpmfield",
"lastRev" : "0",
- "uuid" : "a155ab31-89d9-44d9-94d3-c06fc1fda992",
+ "uuid" : "3c91ae21-f045-413a-b316-c46298982991",
"valid" : true,
"metadata" : {
- "labels" : [ ]
+ "labels" : [ "Spot/WindowsLogs" ]
},
"name" : "ODMWindowsEventLogs9fae6da4-eb7f-4aca-820a-1f561a40e7cc",
- "sdcVersion" : "2.7.1.0",
+ "sdcVersion" : "3.0.0.0",
"sdcId" : "f0ff6a12-61b1-44a6-bc5f-0d7e67d7d482"
},
"metadata" : {
- "labels" : [ ]
+ "labels" : [ "Spot/WindowsLogs" ]
},
"statsAggregatorStage" : {
"instanceName" : "WritetoDPMdirectly_StatsAggregatorStage",
@@ -794,7 +820,8 @@
},
"inputLanes" : [ ],
"outputLanes" : [ ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
},
"startEventStages" : [ {
"instanceName" : "Discard_StartEventStage",
@@ -811,7 +838,8 @@
},
"inputLanes" : [ ],
"outputLanes" : [ ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
} ],
"stopEventStages" : [ {
"instanceName" : "Discard_StopEventStage",
@@ -828,7 +856,8 @@
},
"inputLanes" : [ ],
"outputLanes" : [ ],
- "eventLanes" : [ ]
+ "eventLanes" : [ ],
+ "services" : [ ]
} ],
"valid" : true,
"issues" : {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/5897dc4b/spot-ingest/streamsets/windows/WindowsHTTPEdge.json
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/windows/WindowsHTTPEdge.json b/spot-ingest/streamsets/windows/WindowsHTTPEdge.json
new file mode 100644
index 0000000..9ee7c7f
--- /dev/null
+++ b/spot-ingest/streamsets/windows/WindowsHTTPEdge.json
@@ -0,0 +1,603 @@
+{
+ "pipelineConfig" : {
+ "schemaVersion" : 5,
+ "version" : 7,
+ "pipelineId" : "WindowsHTTPa1bd970b-9591-4e9c-bbb6-f8e109f81068",
+ "title" : "WindowsHTTPEdge",
+ "description" : "",
+ "uuid" : "edb99eec-a1b6-4cce-af04-566805b69308",
+ "configuration" : [ {
+ "name" : "executionMode",
+ "value" : "EDGE"
+ }, {
+ "name" : "deliveryGuarantee",
+ "value" : "AT_LEAST_ONCE"
+ }, {
+ "name" : "startEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "stopEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "shouldRetry",
+ "value" : true
+ }, {
+ "name" : "retryAttempts",
+ "value" : -1
+ }, {
+ "name" : "memoryLimit",
+ "value" : "${jvm:maxMemoryMB() * 0.65}"
+ }, {
+ "name" : "memoryLimitExceeded",
+ "value" : "STOP_PIPELINE"
+ }, {
+ "name" : "notifyOnStates",
+ "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "constants",
+ "value" : [ {
+ "value" : "localhost",
+ "key" : "TARGET_SDC_HOST"
+ }, {
+ "key" : "WINDOWS_HTTP_APP_ID",
+ "value" : "windowsSpotApp"
+ }, {
+ "key" : "HTTP_PORT",
+ "value" : "8000"
+ } ]
+ }, {
+ "name" : "badRecordsHandling",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "errorRecordPolicy",
+ "value" : "ORIGINAL_RECORD"
+ }, {
+ "name" : "workerCount",
+ "value" : 0
+ }, {
+ "name" : "clusterSlaveMemory",
+ "value" : 1024
+ }, {
+ "name" : "clusterSlaveJavaOpts",
+ "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
+ }, {
+ "name" : "clusterLauncherEnv",
+ "value" : [ ]
+ }, {
+ "name" : "mesosDispatcherURL",
+ "value" : null
+ }, {
+ "name" : "hdfsS3ConfDir",
+ "value" : null
+ }, {
+ "name" : "rateLimit",
+ "value" : 0
+ }, {
+ "name" : "maxRunners",
+ "value" : 0
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "sparkConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "statsAggregatorStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
+ }, {
+ "name" : "shouldCreateFailureSnapshot",
+ "value" : true
+ } ],
+ "uiInfo" : {
+ "previewConfig" : {
+ "previewSource" : "CONFIGURED_SOURCE",
+ "batchSize" : 10,
+ "timeout" : 10000,
+ "writeToDestinations" : false,
+ "executeLifecycleEvents" : false,
+ "showHeader" : false,
+ "showFieldType" : true,
+ "rememberMe" : false
+ }
+ },
+ "stages" : [ {
+ "instanceName" : "WindowsEventLog_01",
+ "library" : "streamsets-datacollector-windows-lib",
+ "stageName" : "com_streamsets_pipeline_stage_origin_windows_WindowsEventLogDSource",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "logName",
+ "value" : "System"
+ }, {
+ "name" : "readMode",
+ "value" : "ALL"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Windows Event Log 1",
+ "xPos" : 60,
+ "yPos" : 50,
+ "stageType" : "SOURCE"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ "WindowsEventLog_01OutputLane15038912833790" ],
+ "eventLanes" : [ ],
+ "services" : [ ]
+ }, {
+ "instanceName" : "HTTPClient_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_http_HttpClientDTarget",
+ "stageVersion" : "3",
+ "configuration" : [ {
+ "name" : "conf.dataGeneratorFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvReplaceNewLines",
+ "value" : true
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvReplaceNewLinesString",
+ "value" : " "
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.jsonMode",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.textFieldPath",
+ "value" : "/text"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.textRecordSeparator",
+ "value" : "\\n"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.textFieldMissingAction",
+ "value" : "ERROR"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.textEmptyLineIfNull",
+ "value" : false
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.avroSchemaSource",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.avroSchema",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.registerSchema",
+ "value" : false
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.subjectToRegister",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.includeSchema",
+ "value" : true
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.avroCompression",
+ "value" : "NULL"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.binaryFieldPath",
+ "value" : "/"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.fileNameEL",
+ "value" : null
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.wholeFileExistsAction",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.includeChecksumInTheEvents",
+ "value" : false
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.checksumAlgorithm",
+ "value" : "MD5"
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.xmlPrettyPrint",
+ "value" : true
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.xmlValidateSchema",
+ "value" : false
+ }, {
+ "name" : "conf.dataGeneratorFormatConfig.xmlSchema",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormat",
+ "value" : "SDC_JSON"
+ }, {
+ "name" : "conf.resourceUrl",
+ "value" : "http://${TARGET_SDC_HOST}:${HTTP_PORT}"
+ }, {
+ "name" : "conf.headers",
+ "value" : [ {
+ "key" : "X-SDC-APPLICATION-ID",
+ "value" : "${WINDOWS_HTTP_APP_ID}"
+ } ]
+ }, {
+ "name" : "conf.httpMethod",
+ "value" : "POST"
+ }, {
+ "name" : "conf.methodExpression",
+ "value" : null
+ }, {
+ "name" : "conf.client.transferEncoding",
+ "value" : "CHUNKED"
+ }, {
+ "name" : "conf.client.httpCompression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.connectTimeoutMillis",
+ "value" : 0
+ }, {
+ "name" : "conf.client.readTimeoutMillis",
+ "value" : 0
+ }, {
+ "name" : "conf.client.numThreads",
+ "value" : 1
+ }, {
+ "name" : "conf.client.authType",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.useOAuth2",
+ "value" : false
+ }, {
+ "name" : "conf.client.oauth.consumerKey",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.consumerSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.token",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.tokenSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.credentialsGrantType",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.tokenUrl",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.clientId",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.clientSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.resourceOwnerClientId",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.resourceOwnerClientSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.algorithm",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.oauth2.key",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.jwtClaims",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.transferEncoding",
+ "value" : "BUFFERED"
+ }, {
+ "name" : "conf.client.oauth2.additionalValues",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.basicAuth.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.basicAuth.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.useProxy",
+ "value" : false
+ }, {
+ "name" : "conf.client.proxy.uri",
+ "value" : null
+ }, {
+ "name" : "conf.client.proxy.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.proxy.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.tlsEnabled",
+ "value" : false
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.client.tlsConfig.keyStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.client.tlsConfig.trustStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.client.tlsConfig.useDefaultProtocols",
+ "value" : true
+ }, {
+ "name" : "conf.client.tlsConfig.protocols",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.tlsConfig.useDefaultCiperSuites",
+ "value" : true
+ }, {
+ "name" : "conf.client.tlsConfig.cipherSuites",
+ "value" : [ ]
+ }, {
+ "name" : "conf.singleRequestPerBatch",
+ "value" : true
+ }, {
+ "name" : "conf.rateLimit",
+ "value" : 0
+ }, {
+ "name" : "conf.maxRequestCompletionSecs",
+ "value" : 60
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.requestLoggingConfig.enableRequestLogging",
+ "value" : false
+ }, {
+ "name" : "conf.client.requestLoggingConfig.logLevel",
+ "value" : "FINE"
+ }, {
+ "name" : "conf.client.requestLoggingConfig.verbosity",
+ "value" : "HEADERS_ONLY"
+ }, {
+ "name" : "conf.client.requestLoggingConfig.maxEntitySize",
+ "value" : 0
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "HTTP Client 1",
+ "xPos" : 280,
+ "yPos" : 49,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ "WindowsEventLog_01OutputLane15038912833790" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ],
+ "services" : [ ]
+ } ],
+ "errorStage" : {
+ "instanceName" : "Discard_ErrorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Error Records - Discard",
+ "xPos" : 500,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ],
+ "services" : [ ]
+ },
+ "info" : {
+ "pipelineId" : "WindowsHTTPa1bd970b-9591-4e9c-bbb6-f8e109f81068",
+ "title" : "WindowsHTTPEdge",
+ "description" : "",
+ "created" : 1516140346592,
+ "lastModified" : 1516141342669,
+ "creator" : "natty@dpmfield",
+ "lastModifier" : "natty@dpmfield",
+ "lastRev" : "0",
+ "uuid" : "edb99eec-a1b6-4cce-af04-566805b69308",
+ "valid" : true,
+ "metadata" : {
+ "labels" : [ "Spot/WindowsLogs" ]
+ },
+ "name" : "WindowsHTTPa1bd970b-9591-4e9c-bbb6-f8e109f81068",
+ "sdcVersion" : "3.0.0.0",
+ "sdcId" : "f0ff6a12-61b1-44a6-bc5f-0d7e67d7d482"
+ },
+ "metadata" : {
+ "labels" : [ "Spot/WindowsLogs" ]
+ },
+ "statsAggregatorStage" : {
+ "instanceName" : "WritetoDPMdirectly_StatsAggregatorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stats Aggregator - Write to DPM directly",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ],
+ "services" : [ ]
+ },
+ "startEventStages" : [ {
+ "instanceName" : "Discard_StartEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Start Event - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ],
+ "services" : [ ]
+ } ],
+ "stopEventStages" : [ {
+ "instanceName" : "Discard_StopEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stop Event - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ],
+ "services" : [ ]
+ } ],
+ "valid" : true,
+ "issues" : {
+ "issueCount" : 0,
+ "stageIssues" : { },
+ "pipelineIssues" : [ ]
+ },
+ "previewable" : true
+ },
+ "pipelineRules" : {
+ "schemaVersion" : 3,
+ "version" : 2,
+ "metricsRuleDefinitions" : [ {
+ "id" : "badRecordsAlertID",
+ "alertText" : "High incidence of Error Records",
+ "metricId" : "pipeline.batchErrorRecords.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1512453583562,
+ "valid" : true
+ }, {
+ "id" : "stageErrorAlertID",
+ "alertText" : "High incidence of Stage Errors",
+ "metricId" : "pipeline.batchErrorMessages.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1512453583562,
+ "valid" : true
+ }, {
+ "id" : "idleGaugeID",
+ "alertText" : "Pipeline is Idle",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
+ "condition" : "${time:now() - value() > 120000}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1512453583562,
+ "valid" : true
+ }, {
+ "id" : "batchTimeAlertID",
+ "alertText" : "Batch taking more time to process",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "CURRENT_BATCH_AGE",
+ "condition" : "${value() > 200}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1512453583562,
+ "valid" : true
+ }, {
+ "id" : "memoryLimitAlertID",
+ "alertText" : "Memory limit for pipeline exceeded",
+ "metricId" : "pipeline.memoryConsumed.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > (jvm:maxMemoryMB() * 0.65)}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1512453583562,
+ "valid" : true
+ } ],
+ "dataRuleDefinitions" : [ ],
+ "driftRuleDefinitions" : [ ],
+ "uuid" : "30680747-47d2-4b6c-a25f-0dcba0f3bddb",
+ "configuration" : [ {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ } ],
+ "ruleIssues" : [ ],
+ "configIssues" : [ ]
+ },
+ "libraryDefinitions" : null
+}
\ No newline at end of file
[2/2] incubator-spot git commit: Merge branch 'SPOT-181_ODM' of
https://git-wip-us.apache.org/repos/asf/incubator-spot into pr/133
Posted by na...@apache.org.
Merge branch 'SPOT-181_ODM' of https://git-wip-us.apache.org/repos/asf/incubator-spot into pr/133
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/e62631bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/e62631bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/e62631bb
Branch: refs/heads/SPOT-181_ODM
Commit: e62631bb2f65931d02e8de20fa1d8958a74cc847
Parents: 5897dc4 85d29d4
Author: natedogs911 <na...@gmail.com>
Authored: Tue Jan 23 16:24:40 2018 -0800
Committer: natedogs911 <na...@gmail.com>
Committed: Tue Jan 23 16:24:40 2018 -0800
----------------------------------------------------------------------
.../streamsets/netflow/NetFlowODMandLegacy.json | 1463 ++++++++++++++++++
1 file changed, 1463 insertions(+)
----------------------------------------------------------------------