You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/29 16:55:24 UTC
[1/7] incubator-spot git commit: added ODM branch per SPOT-182
Repository: incubator-spot
Updated Branches:
refs/heads/SPOT-181_ODM 392e2a903 -> d9232593e
added ODM branch per SPOT-182
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/5f251554
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/5f251554
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/5f251554
Branch: refs/heads/SPOT-181_ODM
Commit: 5f251554680bc3936dd6010d6cae836e05031e4b
Parents: 5f25c95
Author: natedogs911 <na...@gmail.com>
Authored: Tue Jun 27 07:40:15 2017 -0700
Committer: natedogs911 <na...@gmail.com>
Committed: Tue Jun 27 07:40:15 2017 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[4/7] incubator-spot git commit: SPOT-227. [Ingest] StreamSets
pipeline configs for Qualys ingest into ODM
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b6bba644/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityEvents.json
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityEvents.json b/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityEvents.json
new file mode 100644
index 0000000..750d5c5
--- /dev/null
+++ b/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityEvents.json
@@ -0,0 +1,1245 @@
+{
+ "pipelineConfig" : {
+ "schemaVersion" : 4,
+ "version" : 7,
+ "pipelineId" : "QualysVulnerabilityEvents4a335924-c445-4388-8b7c-32a4b4726104",
+ "title" : "ODMQualysVulnerabilityEvents",
+ "description" : "",
+ "uuid" : "3f879613-cd25-4152-80e1-c34d7680d0f0",
+ "configuration" : [ {
+ "name" : "executionMode",
+ "value" : "STANDALONE"
+ }, {
+ "name" : "deliveryGuarantee",
+ "value" : "AT_LEAST_ONCE"
+ }, {
+ "name" : "shouldRetry",
+ "value" : true
+ }, {
+ "name" : "retryAttempts",
+ "value" : -1
+ }, {
+ "name" : "memoryLimit",
+ "value" : "${jvm:maxMemoryMB() * 0.65}"
+ }, {
+ "name" : "memoryLimitExceeded",
+ "value" : "STOP_PIPELINE"
+ }, {
+ "name" : "notifyOnStates",
+ "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "constants",
+ "value" : [ {
+ "key" : "QUALYS_API_URL",
+ "value" : ""
+ }, {
+ "key" : "ODM_EVENTS_LOCATION",
+ "value" : ""
+ }, {
+ "key" : "ODM_EVENTS_TABLE_NAME",
+ "value" : ""
+ }, {
+ "key" : "HIVE_URL",
+ "value" : ""
+ } ]
+ }, {
+ "name" : "badRecordsHandling",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "clusterSlaveMemory",
+ "value" : 1024
+ }, {
+ "name" : "clusterSlaveJavaOpts",
+ "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
+ }, {
+ "name" : "clusterLauncherEnv",
+ "value" : [ ]
+ }, {
+ "name" : "mesosDispatcherURL",
+ "value" : null
+ }, {
+ "name" : "hdfsS3ConfDir",
+ "value" : null
+ }, {
+ "name" : "rateLimit",
+ "value" : 0
+ }, {
+ "name" : "maxRunners",
+ "value" : 0
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "statsAggregatorStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
+ }, {
+ "name" : "workerCount",
+ "value" : 0
+ }, {
+ "name" : "startEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "stopEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "errorRecordPolicy",
+ "value" : "ORIGINAL_RECORD"
+ }, {
+ "name" : "sparkConfigs",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "previewConfig" : {
+ "previewSource" : "CONFIGURED_SOURCE",
+ "batchSize" : 10,
+ "timeout" : 10000,
+ "writeToDestinations" : true,
+ "showHeader" : true,
+ "showFieldType" : true,
+ "rememberMe" : false,
+ "executeLifecycleEvents" : true
+ }
+ },
+ "stages" : [ {
+ "instanceName" : "HTTPClient_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_origin_http_HttpClientDSource",
+ "stageVersion" : "13",
+ "configuration" : [ {
+ "name" : "conf.basic.maxBatchSize",
+ "value" : 1000
+ }, {
+ "name" : "conf.basic.maxWaitTime",
+ "value" : 2000
+ }, {
+ "name" : "conf.dataFormatConfig.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.dataFormatConfig.filePatternInArchive",
+ "value" : "*"
+ }, {
+ "name" : "conf.dataFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.dataFormatConfig.removeCtrlChars",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.textMaxLineLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.useCustomDelimiter",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customDelimiter",
+ "value" : "\\r\\n"
+ }, {
+ "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.jsonMaxObjectLen",
+ "value" : 4096
+ }, {
+ "name" : "conf.dataFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "conf.dataFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "conf.dataFormatConfig.csvMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.csvAllowExtraColumns",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
+ "value" : "_extra_"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "conf.dataFormatConfig.csvEnableComments",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvCommentMarker",
+ "value" : "#"
+ }, {
+ "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.csvRecordType",
+ "value" : "LIST_MAP"
+ }, {
+ "name" : "conf.dataFormatConfig.csvSkipStartLines",
+ "value" : 0
+ }, {
+ "name" : "conf.dataFormatConfig.parseNull",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.nullConstant",
+ "value" : "\\\\N"
+ }, {
+ "name" : "conf.dataFormatConfig.xmlRecordElement",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xPathNamespaceContext",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.outputFieldAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xmlMaxObjectLen",
+ "value" : 40960000
+ }, {
+ "name" : "conf.dataFormatConfig.logMode",
+ "value" : "COMMON_LOG_FORMAT"
+ }, {
+ "name" : "conf.dataFormatConfig.logMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.retainOriginalLine",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customLogFormat",
+ "value" : "%h %l %u %t \"%r\" %>s %b"
+ }, {
+ "name" : "conf.dataFormatConfig.regex",
+ "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
+ }, {
+ "name" : "conf.dataFormatConfig.fieldPathsToGroupName",
+ "value" : [ {
+ "fieldPath" : "/",
+ "group" : 1
+ } ]
+ }, {
+ "name" : "conf.dataFormatConfig.grokPatternDefinition",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.grokPattern",
+ "value" : "%{COMMONAPACHELOG}"
+ }, {
+ "name" : "conf.dataFormatConfig.onParseError",
+ "value" : "ERROR"
+ }, {
+ "name" : "conf.dataFormatConfig.maxStackTraceLines",
+ "value" : 50
+ }, {
+ "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.log4jCustomLogFormat",
+ "value" : "%r [%t] %-5p %c %x - %m%n"
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchemaSource",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchema",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "conf.dataFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.isDelimited",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.binaryMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.datagramMode",
+ "value" : "SYSLOG"
+ }, {
+ "name" : "conf.dataFormatConfig.typesDbPath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.convertTime",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.excludeInterval",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.authFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
+ "value" : 8192
+ }, {
+ "name" : "conf.dataFormatConfig.rateLimit",
+ "value" : "-1"
+ }, {
+ "name" : "conf.dataFormatConfig.verifyChecksum",
+ "value" : false
+ }, {
+ "name" : "conf.resourceUrl",
+ "value" : "${QUALYS_API_URL}"
+ }, {
+ "name" : "conf.headers",
+ "value" : [ ]
+ }, {
+ "name" : "conf.httpMethod",
+ "value" : "GET"
+ }, {
+ "name" : "conf.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "conf.requestBody",
+ "value" : null
+ }, {
+ "name" : "conf.defaultRequestContentType",
+ "value" : "application/json"
+ }, {
+ "name" : "conf.client.transferEncoding",
+ "value" : "CHUNKED"
+ }, {
+ "name" : "conf.client.httpCompression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.connectTimeoutMillis",
+ "value" : 0
+ }, {
+ "name" : "conf.client.readTimeoutMillis",
+ "value" : 0
+ }, {
+ "name" : "conf.client.authType",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.useOAuth2",
+ "value" : false
+ }, {
+ "name" : "conf.client.oauth.consumerKey",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.consumerSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.token",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.tokenSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.credentialsGrantType",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.tokenUrl",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.clientId",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.clientSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.resourceOwnerClientId",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.resourceOwnerClientSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.algorithm",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.oauth2.key",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.jwtClaims",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.transferEncoding",
+ "value" : "BUFFERED"
+ }, {
+ "name" : "conf.client.oauth2.additionalValues",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.basicAuth.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.basicAuth.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.useProxy",
+ "value" : false
+ }, {
+ "name" : "conf.client.proxy.uri",
+ "value" : null
+ }, {
+ "name" : "conf.client.proxy.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.proxy.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.tlsEnabled",
+ "value" : false
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.client.tlsConfig.keyStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.client.tlsConfig.trustStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.client.tlsConfig.useDefaultProtocols",
+ "value" : true
+ }, {
+ "name" : "conf.client.tlsConfig.protocols",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.tlsConfig.useDefaultCiperSuites",
+ "value" : true
+ }, {
+ "name" : "conf.client.tlsConfig.cipherSuites",
+ "value" : [ ]
+ }, {
+ "name" : "conf.httpMode",
+ "value" : "POLLING"
+ }, {
+ "name" : "conf.pollingInterval",
+ "value" : 5000
+ }, {
+ "name" : "conf.dataFormat",
+ "value" : "XML"
+ }, {
+ "name" : "conf.responseStatusActionConfigs",
+ "value" : [ {
+ "statusCode" : 500,
+ "action" : "RETRY_EXPONENTIAL_BACKOFF",
+ "backoffInterval" : 1000,
+ "maxNumRetries" : 10
+ } ]
+ }, {
+ "name" : "conf.responseTimeoutActionConfig.action",
+ "value" : "RETRY_IMMEDIATELY"
+ }, {
+ "name" : "conf.responseTimeoutActionConfig.backoffInterval",
+ "value" : 1000
+ }, {
+ "name" : "conf.responseTimeoutActionConfig.maxNumRetries",
+ "value" : 10
+ }, {
+ "name" : "conf.pagination.mode",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.pagination.startAt",
+ "value" : null
+ }, {
+ "name" : "conf.pagination.resultFieldPath",
+ "value" : null
+ }, {
+ "name" : "conf.pagination.keepAllFields",
+ "value" : false
+ }, {
+ "name" : "conf.pagination.rateLimit",
+ "value" : 2000
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Poll Qualys API",
+ "xPos" : 60,
+ "yPos" : 50,
+ "stageType" : "SOURCE"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ "HTTPClient_01OutputLane15059265383830" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldPivoter_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "listPath",
+ "value" : "/RESPONSE[0]/HOST_LIST[0]/HOST"
+ }, {
+ "name" : "copyFields",
+ "value" : true
+ }, {
+ "name" : "newPath",
+ "value" : ""
+ }, {
+ "name" : "saveOriginalFieldName",
+ "value" : false
+ }, {
+ "name" : "originalFieldNamePath",
+ "value" : null
+ }, {
+ "name" : "onStagePreConditionFailure",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Pivot Host List",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "HTTPClient_01OutputLane15059265383830" ],
+ "outputLanes" : [ "FieldPivoter_01OutputLane15024033858660" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldPivoter_02",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "listPath",
+ "value" : "/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION"
+ }, {
+ "name" : "copyFields",
+ "value" : true
+ }, {
+ "name" : "newPath",
+ "value" : null
+ }, {
+ "name" : "saveOriginalFieldName",
+ "value" : false
+ }, {
+ "name" : "originalFieldNamePath",
+ "value" : null
+ }, {
+ "name" : "onStagePreConditionFailure",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Pivot Detection List",
+ "xPos" : 500,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldPivoter_01OutputLane15024033858660" ],
+ "outputLanes" : [ "FieldPivoter_02OutputLane15024040316830" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ {
+ "fieldToSet" : "/dvc_ip4",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/IP[0]/value')}"
+ }, {
+ "fieldToSet" : "/dvc_host",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/ID[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_id",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/QID[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_type",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/TYPE[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_status",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/STATUS[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_severity",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/SEVERITY[0]/value')}"
+ }, {
+ "fieldToSet" : "/event_time",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/LAST_SCAN_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs",
+ "expression" : "${emptyMap()}"
+ }, {
+ "fieldToSet" : "/additional_attrs/port",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/PORT[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/protocol",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/PROTOCOL[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/results",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/RESULTS[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/first_found_datetime",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/FIRST_FOUND_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/last_test_datetime",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/LAST_TEST_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/last_update_datetime",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/LAST_UPDATE_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/last_fixed_datetime",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/LAST_FIXED_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/is_ignored",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/IS_IGNORED[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/is_disabled",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/IS_DISABLED[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/times_found",
+ "expression" : "${record:value('/RESPONSE[0]/HOST_LIST[0]/HOST/DETECTION_LIST[0]/DETECTION/TIMES_FOUND[0]/value')}"
+ } ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ {
+ "attributeToSet" : "p_dvc_vendor",
+ "headerAttributeExpression" : "Qualys"
+ }, {
+ "attributeToSet" : "p_dvc_type",
+ "headerAttributeExpression" : "Qualys"
+ }, {
+ "attributeToSet" : "p_dt",
+ "headerAttributeExpression" : "${time:extractStringFromDate(time:now(),'YYYY-MM-dd')}"
+ } ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Map to ODM",
+ "xPos" : 720,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldPivoter_02OutputLane15024040316830" ],
+ "outputLanes" : [ "ExpressionEvaluator_01OutputLane15024032304190" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldRemover_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldfilter_FieldFilterDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "filterOperation",
+ "value" : "REMOVE"
+ }, {
+ "name" : "fields",
+ "value" : [ "/RESPONSE" ]
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Remove Extra Fields",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_01OutputLane15024032304190" ],
+ "outputLanes" : [ "FieldRemover_01OutputLane15030862999220" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldTypeConverter_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "convertBy",
+ "value" : "BY_FIELD"
+ }, {
+ "name" : "fieldTypeConverterConfigs",
+ "value" : [ {
+ "fields" : [ "/additional_attrs/first_found_datetime", "/additional_attrs/last_fixed_datetime", "/additional_attrs/last_test_datetime", "/additional_attrs/last_update_datetime", "/event_time" ],
+ "targetType" : "DATETIME",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "OTHER",
+ "encoding" : "UTF-8",
+ "otherDateFormat" : "yyyy-MM-dd'T'HH:mm:ss'Z'"
+ }, {
+ "fields" : [ "/additional_attrs/last_fixed_datetime", "/additional_attrs/last_test_datetime", "/additional_attrs/last_update_datetime", "/event_time", "/additional_attrs/first_found_datetime" ],
+ "targetType" : "LONG",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "YYYY_MM_DD",
+ "encoding" : "UTF-8"
+ }, {
+ "fields" : [ "/additional_attrs/first_found_datetime", "/additional_attrs/last_fixed_datetime", "/additional_attrs/last_test_datetime", "/additional_attrs/last_update_datetime" ],
+ "targetType" : "STRING",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "YYYY_MM_DD",
+ "encoding" : "UTF-8"
+ } ]
+ }, {
+ "name" : "wholeTypeConverterConfigs",
+ "value" : [ {
+ "sourceType" : "INTEGER",
+ "targetType" : "INTEGER",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "YYYY_MM_DD",
+ "encoding" : "UTF-8"
+ } ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Convert Datetime Fields",
+ "xPos" : 1160,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldRemover_01OutputLane15030862999220" ],
+ "outputLanes" : [ "FieldTypeConverter_01OutputLane15030863815980" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HadoopFS_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hdfs_HdfsDTarget",
+ "stageVersion" : "4",
+ "configuration" : [ {
+ "name" : "hdfsTargetConfigBean.hdfsUri",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsUser",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsKerberos",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfDir",
+ "value" : "hadoop-conf"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.uniquePrefix",
+ "value" : "sdc-${sdc:id()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.fileNameSuffix",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplateInHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplate",
+ "value" : "${ODM_EVENTS_LOCATION}/p_dvc_vendor=${record:attribute(\"p_dvc_vendor\")}/p_dvc_type=${record:attribute(\"p_dvc_type\")}/p_dt=${record:attribute(\"p_dt\")}"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeDriver",
+ "value" : "${time:now()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.maxRecordsPerFile",
+ "value" : 0
+ }, {
+ "name" : "hdfsTargetConfigBean.maxFileSize",
+ "value" : 256
+ }, {
+ "name" : "hdfsTargetConfigBean.idleTimeout",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "hdfsTargetConfigBean.otherCompression",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.fileType",
+ "value" : "TEXT"
+ }, {
+ "name" : "hdfsTargetConfigBean.keyEl",
+ "value" : "${uuid()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.seqFileCompressionType",
+ "value" : "BLOCK"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsLimit",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.rollIfHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.rollHeaderName",
+ "value" : "roll"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsAction",
+ "value" : "SEND_TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsDirPathTemplate",
+ "value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataFormat",
+ "value" : "AVRO"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsPermissionCheck",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.permissionEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.skipOldTempFileRecovery",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLines",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLinesString",
+ "value" : " "
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.jsonMode",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldPath",
+ "value" : "/text"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textRecordSeparator",
+ "value" : "\\n"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldMissingAction",
+ "value" : "ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textEmptyLineIfNull",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchemaSource",
+ "value" : "INLINE"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchema",
+ "value" : "{\n\n \"namespace\":\"org.apache.spot\",\n \"name\":\"event\",\n \"type\": \"record\",\n \"fields\": [\n {\"name\":\"event_time\",\"type\":[\"null\",\"long\"],\"doc\":\"timestamp of event (UTC)\", \"default\": null},\n {\"name\":\"duration\", \"type\":[\"null\",\"float\"],\"doc\":\"Time duration (milliseconds)\", \"default\": null},\n {\"name\":\"event_id\", \"type\":[\"null\",\"string\"],\"doc\":\"Unique identifier for event\", \"default\": null},\n {\"name\":\"name\", \"type\":[\"null\",\"string\"],\"doc\":\"Name of event\", \"default\": null},\n {\"name\":\"org\", \"type\":[\"null\",\"string\"],\"doc\":\"Organization\", \"default\": null},\n {\"name\":\"type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type information\", \"default\": null},\n {\"name\":\"n_proto\", \"type\":[\"null\",\"string\"],\"doc\":\"Network protocol of event\", \"default\": null},\n {\"name\":\"a_proto\"
, \"type\":[\"null\",\"string\"],\"doc\":\"Application protocol of event\", \"default\": null},\n {\"name\":\"msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Message (details of action taken on object)\", \"default\": null},\n {\"name\":\"mac\", \"type\":[\"null\",\"string\"],\"doc\":\"MAC address\", \"default\": null},\n {\"name\":\"severity\", \"type\":[\"null\",\"string\"],\"doc\":\"Severity of event\", \"default\": null},\n {\"name\":\"raw\", \"type\":[\"null\",\"string\"],\"doc\":\"Raw text message of entire event\", \"default\": null},\n {\"name\":\"risk\", \"type\":[\"null\",\"float\"],\"doc\":\"Risk score\", \"default\": null},\n {\"name\":\"code\", \"type\":[\"null\",\"string\"],\"doc\":\"Response or error code\", \"default\": null},\n {\"name\":\"category\", \"type\":[\"null\",\"string\"],\"doc\":\"Event category\", \"default\": null},\n {\"name\":\"query\", \"type\":[\"null\",\"string\"],\"doc\":\"Que
ry (DNS query, URI query, SQL query, etc.)\", \"default\": null},\n {\"name\":\"service\", \"type\":[\"null\",\"string\"],\"doc\":\"(i.e. service name, type of service)\", \"default\": null},\n {\"name\":\"state\", \"type\":[\"null\",\"string\"],\"doc\":\"State of object\", \"default\": null},\n {\"name\":\"in_bytes\", \"type\":[\"null\",\"int\"],\"doc\":\"Bytes in\", \"default\": null},\n {\"name\":\"out_bytes\", \"type\":[\"null\",\"int\"],\"doc\":\"Bytes out\", \"default\": null},\n {\"name\":\"xref\", \"type\":[\"null\",\"string\"],\"doc\":\"External reference to public description\", \"default\": null},\n {\"name\":\"version\", \"type\":[\"null\",\"string\"],\"doc\":\"Version\", \"default\": null},\n {\"name\":\"additional_attrs\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null, \"doc\":\"Additional attributes of the event\", \"default\": null},\n {\"name\":\"dvc_time\
", \"type\":[\"null\",\"long\"],\"doc\":\"UTC timestamp from device where event/alert originates or is received\", \"default\": null},\n {\"name\":\"dvc_ip4\", \"type\":[\"null\",\"string\"],\"doc\":\"IP address of device\", \"default\": null},\n {\"name\":\"dvc_ip6\", \"type\":[\"null\",\"string\"],\"doc\":\"IP address of device\", \"default\": null},\n {\"name\":\"dvc_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Hostname of device\", \"default\": null},\n {\"name\":\"dvc_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Device type that generated the log\", \"default\": null},\n {\"name\":\"dvc_vendor\", \"type\":[\"null\",\"string\"],\"doc\":\"Vendor\", \"default\": null},\n {\"name\":\"dvc_fwd_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Forwarded from device\", \"default\": null},\n {\"name\":\"dvc_fwd_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Forwarded from device\", \"default\": null},\n {\"name\":\"dvc_version\"
, \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"src_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Source ip address of event\", \"default\": null},\n {\"name\":\"src_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Source ip address of event\", \"default\": null},\n {\"name\":\"src_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Source FQDN of event\", \"default\": null},\n {\"name\":\"src_domain\", \"type\":[\"null\",\"string\"],\"doc\":\"Domain name of source address\", \"default\": null},\n {\"name\":\"src_port\", \"type\":[\"null\",\"int\"],\"doc\":\"Source port of event\", \"default\": null},\n {\"name\":\"src_country_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country code\", \"default\": null},\n {\"name\":\"src_country_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country name\", \"default\": null},\n {\"name\":\"src_region\", \"type\":[\"null\",\"string\"],\"doc\"
:\"Source region\", \"default\": null},\n {\"name\":\"src_city\", \"type\":[\"null\",\"string\"],\"doc\":\"Source city\", \"default\": null},\n {\"name\":\"src_lat\", \"type\":[\"null\",\"int\"],\"doc\":\"Source latitude\", \"default\": null},\n {\"name\":\"src_long\", \"type\":[\"null\",\"int\"],\"doc\":\"Source longitude\", \"default\": null},\n {\"name\":\"dst_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Destination ip address of event\", \"default\": null},\n {\"name\":\"dst_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Destination ip address of event\", \"default\": null},\n {\"name\":\"dst_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Destination FQDN of event\", \"default\": null},\n {\"name\":\"dst_domain\", \"type\":[\"null\",\"string\"],\"doc\":\"Domain name of destination address\", \"default\": null},\n {\"name\":\"dst_port\", \"type\":[\"null\",\"int\"],\"doc\":\"Destination port of event\", \"default\": null},
\n {\"name\":\"dst_country_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country code\", \"default\": null},\n {\"name\":\"dst_country_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country name\", \"default\": null},\n {\"name\":\"dst_region\", \"type\":[\"null\",\"string\"],\"doc\":\"Source region\", \"default\": null},\n {\"name\":\"dst_city\", \"type\":[\"null\",\"string\"],\"doc\":\"Source city\", \"default\": null},\n {\"name\":\"dst_lat\", \"type\":[\"null\",\"int\"],\"doc\":\"Source latitude\", \"default\": null},\n {\"name\":\"dst_long\", \"type\":[\"null\",\"int\"],\"doc\":\"Source longitude\", \"default\": null},\n {\"name\":\"src_asn\", \"type\":[\"null\",\"int\"],\"doc\":\"Autonomous system number\", \"default\": null},\n {\"name\":\"dst_asn\", \"type\":[\"null\",\"int\"],\"doc\":\"Autonomous system number\", \"default\": null},\n {\"name\":\"net_direction\", \"type\":[\"null\",\"string\"]
,\"doc\":\"Direction\", \"default\": null},\n {\"name\":\"net_flags\", \"type\":[\"null\",\"string\"],\"doc\":\"TCP flags\", \"default\": null},\n {\"name\":\"file_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Filename from event\", \"default\": null},\n {\"name\":\"file_path\", \"type\":[\"null\",\"string\"],\"doc\":\"File path\", \"default\": null},\n {\"name\":\"file_atime\", \"type\":[\"null\",\"long\"],\"doc\":\"Timestamp (UTC) of file access\", \"default\": null},\n {\"name\":\"file_acls\", \"type\":[\"null\",\"string\"],\"doc\":\"File permissions\", \"default\": null},\n {\"name\":\"file_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of file\", \"default\": null},\n {\"name\":\"file_size\", \"type\":[\"null\",\"int\"],\"doc\":\"Size of file in bytes\", \"default\": null},\n {\"name\":\"file_desc\", \"type\":[\"null\",\"string\"],\"doc\":\"Description of file\", \"default\": null},\n {\"name\":\"file_hash
\", \"type\":[\"null\",\"string\"],\"doc\":\"Hash of file\", \"default\": null},\n {\"name\":\"file_hash_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of hash\", \"default\": null},\n {\"name\":\"end_object\", \"type\":[\"null\",\"string\"],\"doc\":\"File/Process/ Registry\", \"default\": null},\n {\"name\":\"end_action\", \"type\":[\"null\",\"string\"],\"doc\":\"Action taken on object (open/delete/ edit)\", \"default\": null},\n {\"name\":\"end_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Message (details of action taken on object)\", \"default\": null},\n {\"name\":\"end_app\", \"type\":[\"null\",\"string\"],\"doc\":\"Application\", \"default\": null},\n {\"name\":\"end_location\", \"type\":[\"null\",\"string\"],\"doc\":\"Location\", \"default\": null},\n {\"name\":\"end_proc\", \"type\":[\"null\",\"string\"],\"doc\":\"Process\", \"default\": null},\n {\"name\":\"user_name\", \"type\":[\"null\",\"string\"],\"doc\":
\"username from event\", \"default\": null},\n {\"name\":\"src_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"dst_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"user_email\", \"type\":[\"null\",\"string\"],\"doc\":\"Email address\", \"default\": null},\n {\"name\":\"user_id\", \"type\":[\"null\",\"string\"],\"doc\":\"userid\", \"default\": null},\n {\"name\":\"user_loc\", \"type\":[\"null\",\"string\"],\"doc\":\"location\", \"default\": null},\n {\"name\":\"user_desc\", \"type\":[\"null\",\"string\"],\"doc\":\"Description of user\", \"default\": null},\n {\"name\":\"dns_class\", \"type\":[\"null\",\"string\"],\"doc\":\"DNS class\", \"default\": null},\n {\"name\":\"dns_len\", \"type\":[\"null\",\"int\"],\"doc\":\"DNS frame length\", \"default\": null},\n {\"name\":\"dns_query\", \"type\":[\"null
\",\"string\"],\"doc\":\"Requested DNS query\", \"default\": null},\n {\"name\":\"dns_response_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Response code\", \"default\": null},\n {\"name\":\"dns_answers\", \"type\":[\"null\",\"string\"],\"doc\":\"Response to DNS Query\", \"default\": null},\n {\"name\":\"dns_type\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"prx_category\", \"type\":[\"null\",\"string\"],\"doc\":\"Event category\", \"default\": null},\n {\"name\":\"prx_browser\", \"type\":[\"null\",\"string\"],\"doc\":\"Web browser\", \"default\": null},\n {\"name\":\"prx_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Error or response code\", \"default\": null},\n {\"name\":\"prx_referrer\", \"type\":[\"null\",\"string\"],\"doc\":\"Referrer\", \"default\": null},\n {\"name\":\"prx_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Requested URI\", \"default\": null},\n {\"name\":\
"prx_filter_rule\", \"type\":[\"null\",\"string\"],\"doc\":\"Applied filter or rule\", \"default\": null},\n {\"name\":\"prx_filter_result\", \"type\":[\"null\",\"string\"],\"doc\":\"Result of applied filter or rule\", \"default\": null},\n {\"name\":\"prx_query\", \"type\":[\"null\",\"string\"],\"doc\":\"URI query\", \"default\": null},\n {\"name\":\"prx_action\", \"type\":[\"null\",\"string\"],\"doc\":\"Action taken on object\", \"default\": null},\n {\"name\":\"prx_method\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP method\", \"default\": null},\n {\"name\":\"prx_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of request\", \"default\": null},\n {\"name\":\"http_request_method\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP method\", \"default\": null},\n {\"name\":\"http_request_uri\", \"type\":[\"null\",\"string\"],\"doc\":\"Requested URI\", \"default\": null},\n {\"name\":\"http_request_body_len\", \"type\":[\"
null\",\"int\"],\"doc\":\"Length of request body\", \"default\": null},\n {\"name\":\"http_request_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"http_request_password\", \"type\":[\"null\",\"string\"],\"doc\":\"Password from event\", \"default\": null},\n {\"name\":\"http_request_proxied\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"http_request_headers\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP request headers\", \"default\": null},\n {\"name\":\"http_response_status_code\", \"type\":[\"null\",\"int\"],\"doc\":\"HTTP response status code\", \"default\": null},\n {\"name\":\"http_response_status_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP response status message\", \"default\": null},\n {\"name\":\"http_response_body_len\", \"type\":[\"null\",\"string\"],\"doc\":\"Length of response body\", \"default\": null},\n
{\"name\":\"http_response_info_code\", \"type\":[\"null\",\"int\"],\"doc\":\"HTTP response info code\", \"default\": null},\n {\"name\":\"http_response_info_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP response info message\", \"default\": null},\n {\"name\":\"http_response_resp_fuids\", \"type\":[\"null\",\"string\"],\"doc\":\"Response FUIDS\", \"default\": null},\n {\"name\":\"http_response_mime_types\", \"type\":[\"null\",\"string\"],\"doc\":\"Mime types\", \"default\": null},\n {\"name\":\"http_response_headers\", \"type\":[\"null\",\"string\"],\"doc\":\"Response headers\", \"default\": null},\n {\"name\":\"smtp_trans_depth\", \"type\":[\"null\",\"int\"],\"doc\":\"Depth of email into SMTP exchange\", \"default\": null},\n {\"name\":\"smtp_headers_helo\", \"type\":[\"null\",\"string\"],\"doc\":\"Helo header\", \"default\": null},\n {\"name\":\"smtp_headers_mailfrom\", \"type\":[\"null\",\"string\"],\"doc\":\"Mailfrom head
er\", \"default\": null},\n {\"name\":\"smtp_headers_rcptto\", \"type\":[\"null\",\"string\"],\"doc\":\"Rcptto header\", \"default\": null},\n {\"name\":\"smtp_headers_date\", \"type\":[\"null\",\"string\"],\"doc\":\"Header date\", \"default\": null},\n {\"name\":\"smtp_headers_from\", \"type\":[\"null\",\"string\"],\"doc\":\"From header\", \"default\": null},\n {\"name\":\"smtp_headers_to\", \"type\":[\"null\",\"string\"],\"doc\":\"To header\", \"default\": null},\n {\"name\":\"smtp_headers_reply_to\", \"type\":[\"null\",\"string\"],\"doc\":\"Reply to header\", \"default\": null},\n {\"name\":\"smtp_headers_msg_id\", \"type\":[\"null\",\"string\"],\"doc\":\"Message ID\", \"default\": null},\n {\"name\":\"smtp_headers_in_reply_to\", \"type\":[\"null\",\"string\"],\"doc\":\"In reply to header\", \"default\": null},\n {\"name\":\"smpt_headers_subject\", \"type\":[\"null\",\"string\"],\"doc\":\"Subject\", \"default\": null},\n
{\"name\":\"smtp_headers_x_originating_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Originating IP address\", \"default\": null},\n {\"name\":\"smtp_headers_first_received\", \"type\":[\"null\",\"string\"],\"doc\":\"First to receive message\", \"default\": null},\n {\"name\":\"smtp_headers_second_received\", \"type\":[\"null\",\"string\"],\"doc\":\"Second to receive message\", \"default\": null},\n {\"name\":\"smtp_last_reply\", \"type\":[\"null\",\"string\"],\"doc\":\"Last reply in message chain\", \"default\": null},\n {\"name\":\"smtp_path\", \"type\":[\"null\",\"string\"],\"doc\":\"Path of message\", \"default\": null},\n {\"name\":\"smtp_user_agent\", \"type\":[\"null\",\"string\"],\"doc\":\"User agent\", \"default\": null},\n {\"name\":\"smtp_tls\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Indication of TLS use\", \"default\": null},\n {\"name\":\"smtp_is_webmail\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Indication of w
ebmail\", \"default\": null},\n {\"name\":\"ftp_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Username\", \"default\": null},\n {\"name\":\"ftp_password\", \"type\":[\"null\",\"string\"],\"doc\":\"Password\", \"default\": null},\n {\"name\":\"ftp_command\", \"type\":[\"null\",\"string\"],\"doc\":\"FTP command\", \"default\": null},\n {\"name\":\"ftp_arg\", \"type\":[\"null\",\"string\"],\"doc\":\"Argument\", \"default\": null},\n {\"name\":\"ftp_mime_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Mime type\", \"default\": null},\n {\"name\":\"ftp_file_size\", \"type\":[\"null\",\"int\"],\"doc\":\"File size\", \"default\": null},\n {\"name\":\"ftp_reply_code\", \"type\":[\"null\",\"int\"],\"doc\":\"Reply code\", \"default\": null},\n {\"name\":\"ftp_reply_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Reply message\", \"default\": null},\n {\"name\":\"ftp_data_channel_passive\", \"type\":[\"null\",\"boolean\"],
\"doc\":\"Passive data channel?\", \"default\": null},\n {\"name\":\"ftp_data_channel_rsp_p\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_cwd\", \"type\":[\"null\",\"string\"],\"doc\":\"Current working directory\", \"default\": null},\n {\"name\":\"ftp_cmdarg_ts\", \"type\":[\"null\",\"float\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_cmdarg_cmd\", \"type\":[\"null\",\"string\"],\"doc\":\"Command\", \"default\": null},\n {\"name\":\"ftp_cmdarg_arg\", \"type\":[\"null\",\"string\"],\"doc\":\"Command argument\", \"default\": null},\n {\"name\":\"ftp_cmdarg_seq\", \"type\":[\"null\",\"int\"],\"doc\":\"Sequence\", \"default\": null},\n {\"name\":\"ftp_pending_commands\", \"type\":[\"null\",\"string\"],\"doc\":\"Pending commands\", \"default\": null},\n {\"name\":\"ftp_is_passive\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Passive mode enabled\", \"default\": null},\n
{\"name\":\"ftp_fuid\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_last_auth_requested\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_community\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_bulk_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_responses\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_set_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_display_string\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_up_si
nce\", \"type\":[\"null\",\"float\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_cipher\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_curve\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_server_name\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_resumed\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_next_protocol\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_established\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_cert_chain_fuids\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_client_cert_chain_fuids\", \"type\":[\"null\
",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_auth_success\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_client\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_server\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_cipher_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_mac_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_compression_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_key_exchange_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_host_key_algorithm\", \"type\":[\"null\",
\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_assigned_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_mac\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_lease_time\", \"type\":[\"null\",\"double\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_user\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_nickname\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_command\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_value\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_additional_data\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_in_packets\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n
{\"name\":\"flow_out_packets\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_conn_state\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_history\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_src_dscp\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_dst_dscp\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_input\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_output\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_id\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_title\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_type\", \"type\":[\"null\",\"str
ing\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_status\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_severity\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null}\n ],\n \"doc\": \"A view schema for storing Apache Spot Event data.\"\n }"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.registerSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subjectToRegister",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroCompression",
+ "value" : "NULL"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.binaryFieldPath",
+ "value" : "/"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.fileNameEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.wholeFileExistsAction",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.includeChecksumInTheEvents",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.checksumAlgorithm",
+ "value" : "MD5"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlPrettyPrint",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlValidateSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlSchema",
+ "value" : null
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Hadoop FS 1",
+ "xPos" : 1380,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ "FieldTypeConverter_01OutputLane15030863815980" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ "HadoopFS_01_EventLane" ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_02",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ {
+ "attributeToSet" : "p_dvc_vendor",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -4), \".*=(.*)\", 1)}"
+ }, {
+ "attributeToSet" : "p_dvc_type",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -3), \".*=(.*)\", 1)}"
+ }, {
+ "attributeToSet" : "p_dt",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -2), \".*=(.*)\", 1)}"
+ } ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ {
+ "fieldToSet" : "/"
+ } ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Extract Partition Values",
+ "xPos" : 1600,
+ "yPos" : 200,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "HadoopFS_01_EventLane" ],
+ "outputLanes" : [ "ExpressionEvaluator_02OutputLane15059712540750" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HiveQuery_01",
+ "library" : "streamsets-datacollector-cdh_5_7-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "config.hiveConfigBean.hiveJDBCUrl",
+ "value" : "${HIVE_URL}"
+ }, {
+ "name" : "config.hiveConfigBean.hiveJDBCDriver",
+ "value" : "org.apache.hive.jdbc.HiveDriver"
+ }, {
+ "name" : "config.hiveConfigBean.confDir",
+ "value" : "hive-conf"
+ }, {
+ "name" : "config.hiveConfigBean.additionalConfigProperties",
+ "value" : [ ]
+ }, {
+ "name" : "config.queries",
+ "value" : [ "ALTER TABLE ${ODM_EVENTS_TABLE_NAME} ADD IF NOT EXISTS PARTITION (p_dvc_type='${record:attribute(\"p_dvc_type\")}', p_dvc_vendor='${record:attribute(\"p_dvc_vendor\")}', p_dt='${record:attribute(\"p_dt\")}')" ]
+ }, {
+ "name" : "config.stopOnQueryFailure",
+ "value" : true
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Add Partition to Avro Table",
+ "xPos" : 1820,
+ "yPos" : 200,
+ "stageType" : "EXECUTOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_02OutputLane15059712540750" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "errorStage" : {
+ "instanceName" : "Discard_ErrorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Error Records - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "info" : {
+ "pipelineId" : "QualysVulnerabilityEvents4a335924-c445-4388-8b7c-32a4b4726104",
+ "title" : "ODMQualysVulnerabilityEvents",
+ "description" : "",
+ "created" : 1505937299104,
+ "lastModified" : 1505971516410,
+ "creator" : "natty@dpmfield",
+ "lastModifier" : "natty@dpmfield",
+ "lastRev" : "0",
+ "uuid" : "3f879613-cd25-4152-80e1-c34d7680d0f0",
+ "valid" : true,
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "name" : "QualysVulnerabilityEvents4a335924-c445-4388-8b7c-32a4b4726104",
+ "sdcVersion" : "2.7.1.0",
+ "sdcId" : "f0ff6a12-61b1-44a6-bc5f-0d7e67d7d482"
+ },
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "statsAggregatorStage" : {
+ "instanceName" : "WritetoDPMdirectly_StatsAggregatorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stats Aggregator - Write to DPM directly",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "startEventStages" : [ {
+ "instanceName" : "Discard_StartEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Start Event - Discard",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "stopEventStages" : [ {
+ "instanceName" : "Discard_StopEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stop Event - Discard",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "valid" : true,
+ "issues" : {
+ "issueCount" : 0,
+ "stageIssues" : { },
+ "pipelineIssues" : [ ]
+ },
+ "previewable" : true
+ },
+ "pipelineRules" : {
+ "schemaVersion" : 3,
+ "version" : 2,
+ "metricsRuleDefinitions" : [ {
+ "id" : "badRecordsAlertID",
+ "alertText" : "High incidence of Error Records",
+ "metricId" : "pipeline.batchErrorRecords.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "stageErrorAlertID",
+ "alertText" : "High incidence of Stage Errors",
+ "metricId" : "pipeline.batchErrorMessages.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "idleGaugeID",
+ "alertText" : "Pipeline is Idle",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
+ "condition" : "${time:now() - value() > 120000}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "batchTimeAlertID",
+ "alertText" : "Batch taking more time to process",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "CURRENT_BATCH_AGE",
+ "condition" : "${value() > 200}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "memoryLimitAlertID",
+ "alertText" : "Memory limit for pipeline exceeded",
+ "metricId" : "pipeline.memoryConsumed.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > (jvm:maxMemoryMB() * 0.65)}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ } ],
+ "dataRuleDefinitions" : [ ],
+ "driftRuleDefinitions" : [ ],
+ "uuid" : "e77f5c87-a3d0-4f6d-99c1-c636beb6fb6d",
+ "configuration" : [ {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ } ],
+ "ruleIssues" : [ ],
+ "configIssues" : [ ]
+ },
+ "libraryDefinitions" : null
+}
\ No newline at end of file
[2/7] incubator-spot git commit: SPOT-229. [Ingest] StreamSets
pipeline configs for Centrify event logs
Posted by na...@apache.org.
SPOT-229. [Ingest] StreamSets pipeline configs for Centrify event logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/fe5862c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/fe5862c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/fe5862c4
Branch: refs/heads/SPOT-181_ODM
Commit: fe5862c4cd57374d9eeee4ce68aa92fdb6a50344
Parents: 5f25155
Author: Jon Natkins <na...@streamsets.com>
Authored: Wed Sep 20 22:46:15 2017 -0700
Committer: Jon Natkins <na...@streamsets.com>
Committed: Thu Sep 21 13:43:17 2017 -0700
----------------------------------------------------------------------
.../ODMCentrifyIdentityPlatformEventTCP.json | 1096 ++++++++++++++++++
1 file changed, 1096 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fe5862c4/spot-ingest/streamsets/centrify/ODMCentrifyIdentityPlatformEventTCP.json
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/centrify/ODMCentrifyIdentityPlatformEventTCP.json b/spot-ingest/streamsets/centrify/ODMCentrifyIdentityPlatformEventTCP.json
new file mode 100644
index 0000000..5dd57d2
--- /dev/null
+++ b/spot-ingest/streamsets/centrify/ODMCentrifyIdentityPlatformEventTCP.json
@@ -0,0 +1,1096 @@
+{
+ "pipelineConfig" : {
+ "schemaVersion" : 4,
+ "version" : 7,
+ "pipelineId" : "CentrifyUserEventTCPf5cff562-2e3b-46c7-9dd1-295df3c10b74",
+ "title" : "ODMCentrifyIdentityPlatformEventTCP",
+ "description" : "",
+ "uuid" : "1fac1baa-3ae7-4aa0-a289-80bc39552100",
+ "configuration" : [ {
+ "name" : "executionMode",
+ "value" : "STANDALONE"
+ }, {
+ "name" : "deliveryGuarantee",
+ "value" : "AT_LEAST_ONCE"
+ }, {
+ "name" : "shouldRetry",
+ "value" : true
+ }, {
+ "name" : "retryAttempts",
+ "value" : -1
+ }, {
+ "name" : "memoryLimit",
+ "value" : "${jvm:maxMemoryMB() * 0.65}"
+ }, {
+ "name" : "memoryLimitExceeded",
+ "value" : "STOP_PIPELINE"
+ }, {
+ "name" : "notifyOnStates",
+ "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "constants",
+ "value" : [ {
+ "key" : "CENTRIFY_SYSLOG_PORT",
+ "value" : ""
+ }, {
+ "key" : "ODM_EVENTS_LOCATION",
+ "value" : ""
+ }, {
+ "key" : "ODM_EVENTS_TABLE_NAME",
+ "value" : ""
+ }, {
+ "key" : "HIVE_URL",
+ "value" : ""
+ } ]
+ }, {
+ "name" : "badRecordsHandling",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "clusterSlaveMemory",
+ "value" : 1024
+ }, {
+ "name" : "clusterSlaveJavaOpts",
+ "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
+ }, {
+ "name" : "clusterLauncherEnv",
+ "value" : [ ]
+ }, {
+ "name" : "mesosDispatcherURL",
+ "value" : null
+ }, {
+ "name" : "hdfsS3ConfDir",
+ "value" : null
+ }, {
+ "name" : "rateLimit",
+ "value" : 0
+ }, {
+ "name" : "maxRunners",
+ "value" : 0
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "statsAggregatorStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
+ }, {
+ "name" : "workerCount",
+ "value" : 0
+ }, {
+ "name" : "startEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "stopEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "errorRecordPolicy",
+ "value" : "ORIGINAL_RECORD"
+ }, {
+ "name" : "sparkConfigs",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "previewConfig" : {
+ "previewSource" : "CONFIGURED_SOURCE",
+ "batchSize" : 10,
+ "timeout" : 10000,
+ "writeToDestinations" : true,
+ "showHeader" : true,
+ "showFieldType" : true,
+ "rememberMe" : false,
+ "executeLifecycleEvents" : true
+ }
+ },
+ "stages" : [ {
+ "instanceName" : "TCPServer_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_origin_tcp_TCPServerDSource",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "conf.dataFormat",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.dataFormatConfig.filePatternInArchive",
+ "value" : "*"
+ }, {
+ "name" : "conf.dataFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.dataFormatConfig.removeCtrlChars",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.textMaxLineLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.useCustomDelimiter",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customDelimiter",
+ "value" : "\\r\\n"
+ }, {
+ "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.jsonContent",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "conf.dataFormatConfig.jsonMaxObjectLen",
+ "value" : 4096
+ }, {
+ "name" : "conf.dataFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "conf.dataFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "conf.dataFormatConfig.csvMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.csvAllowExtraColumns",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
+ "value" : "_extra_"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "conf.dataFormatConfig.csvEnableComments",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvCommentMarker",
+ "value" : "#"
+ }, {
+ "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.csvRecordType",
+ "value" : "LIST_MAP"
+ }, {
+ "name" : "conf.dataFormatConfig.csvSkipStartLines",
+ "value" : 0
+ }, {
+ "name" : "conf.dataFormatConfig.parseNull",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.nullConstant",
+ "value" : "\\\\N"
+ }, {
+ "name" : "conf.dataFormatConfig.xmlRecordElement",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xPathNamespaceContext",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.outputFieldAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xmlMaxObjectLen",
+ "value" : 4096
+ }, {
+ "name" : "conf.dataFormatConfig.logMode",
+ "value" : "COMMON_LOG_FORMAT"
+ }, {
+ "name" : "conf.dataFormatConfig.logMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.retainOriginalLine",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customLogFormat",
+ "value" : "%h %l %u %t \"%r\" %>s %b"
+ }, {
+ "name" : "conf.dataFormatConfig.regex",
+ "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
+ }, {
+ "name" : "conf.dataFormatConfig.fieldPathsToGroupName",
+ "value" : [ {
+ "fieldPath" : "/",
+ "group" : 1
+ } ]
+ }, {
+ "name" : "conf.dataFormatConfig.grokPatternDefinition",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.grokPattern",
+ "value" : "%{COMMONAPACHELOG}"
+ }, {
+ "name" : "conf.dataFormatConfig.onParseError",
+ "value" : "ERROR"
+ }, {
+ "name" : "conf.dataFormatConfig.maxStackTraceLines",
+ "value" : 50
+ }, {
+ "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.log4jCustomLogFormat",
+ "value" : "%r [%t] %-5p %c %x - %m%n"
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchemaSource",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchema",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "conf.dataFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.isDelimited",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.binaryMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.datagramMode",
+ "value" : "SYSLOG"
+ }, {
+ "name" : "conf.dataFormatConfig.typesDbPath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.convertTime",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.excludeInterval",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.authFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
+ "value" : 8192
+ }, {
+ "name" : "conf.dataFormatConfig.rateLimit",
+ "value" : "-1"
+ }, {
+ "name" : "conf.dataFormatConfig.verifyChecksum",
+ "value" : false
+ }, {
+ "name" : "conf.tlsConfigBean.tlsEnabled",
+ "value" : false
+ }, {
+ "name" : "conf.tlsConfigBean.keyStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.tlsConfigBean.keyStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.tlsConfigBean.keyStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.tlsConfigBean.keyStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.tlsConfigBean.useDefaultProtocols",
+ "value" : true
+ }, {
+ "name" : "conf.tlsConfigBean.protocols",
+ "value" : [ ]
+ }, {
+ "name" : "conf.tlsConfigBean.useDefaultCiperSuites",
+ "value" : true
+ }, {
+ "name" : "conf.tlsConfigBean.cipherSuites",
+ "value" : [ ]
+ }, {
+ "name" : "conf.ports",
+ "value" : [ "${CENTRIFY_SYSLOG_PORT}" ]
+ }, {
+ "name" : "conf.enableEpoll",
+ "value" : false
+ }, {
+ "name" : "conf.numThreads",
+ "value" : 1
+ }, {
+ "name" : "conf.tcpMode",
+ "value" : "SYSLOG"
+ }, {
+ "name" : "conf.syslogFramingMode",
+ "value" : "NON_TRANSPARENT_FRAMING"
+ }, {
+ "name" : "conf.nonTransparentFramingSeparatorCharStr",
+ "value" : "\\u000A"
+ }, {
+ "name" : "conf.syslogCharset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.recordSeparatorStr",
+ "value" : "\\u000A"
+ }, {
+ "name" : "conf.lengthFieldCharset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.batchSize",
+ "value" : 1000
+ }, {
+ "name" : "conf.maxWaitTime",
+ "value" : 1000
+ }, {
+ "name" : "conf.maxMessageSize",
+ "value" : 4096000
+ }, {
+ "name" : "conf.ackMessageCharset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "conf.recordProcessedAckMessage",
+ "value" : null
+ }, {
+ "name" : "conf.batchCompletedAckMessage",
+ "value" : null
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Centrify Syslog Listener",
+ "xPos" : 60,
+ "yPos" : 50,
+ "stageType" : "SOURCE"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ "TCPServer_01OutputLane15059705973970" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "LogParser_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_logparser_LogParserDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "fieldPathToParse",
+ "value" : "/remaining"
+ }, {
+ "name" : "removeCtrlChars",
+ "value" : false
+ }, {
+ "name" : "parsedFieldPath",
+ "value" : "/parsed_fields"
+ }, {
+ "name" : "logMode",
+ "value" : "GROK"
+ }, {
+ "name" : "customLogFormat",
+ "value" : "%h %l %u %t \"%r\" %>s %b"
+ }, {
+ "name" : "regex",
+ "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
+ }, {
+ "name" : "fieldPathsToGroupName",
+ "value" : [ {
+ "fieldPath" : "/",
+ "group" : 1
+ } ]
+ }, {
+ "name" : "grokPatternDefinition",
+ "value" : "REMAINING %{PROG:program} %{POSINT:pid} - - %{WORD:log_level} %{HOSTNAME:product}\\|%{HOSTNAME:category}\\|%{HOSTNAME:name}\\|%{GREEDYDATA:remaining}"
+ }, {
+ "name" : "grokPattern",
+ "value" : "%{REMAINING}"
+ }, {
+ "name" : "enableLog4jCustomLogFormat",
+ "value" : false
+ }, {
+ "name" : "log4jCustomLogFormat",
+ "value" : "%r [%t] %-5p %c %x - %m%n"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Parse CIP Preamble",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "TCPServer_01OutputLane15059705973970" ],
+ "outputLanes" : [ "LogParser_01OutputLane15059581002910" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldMerger_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldmerger_FieldMergerDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "mergeMapping",
+ "value" : [ {
+ "fromField" : "/parsed_fields",
+ "toField" : "/"
+ } ]
+ }, {
+ "name" : "onStagePreConditionFailure",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "overwriteExisting",
+ "value" : true
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Flatten",
+ "xPos" : 500,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "LogParser_01OutputLane15059581002910" ],
+ "outputLanes" : [ "FieldMerger_01OutputLane15059661333060" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "JythonEvaluator_01",
+ "library" : "streamsets-datacollector-jython_2_7-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_jython_JythonDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "processingMode",
+ "value" : "BATCH"
+ }, {
+ "name" : "initScript",
+ "value" : ""
+ }, {
+ "name" : "script",
+ "value" : "import re\n\nfor record in records:\n try:\n for field in re.finditer(r\"(\\b\\w+=.*?(?=\\s\\w+=|$))\", record.value['remaining']):\n split_field = field.group(0).split(\"=\")\n record.value[split_field[0]] = split_field[1][1:-1]\n output.write(record)\n\n except Exception as e:\n # Send record to error\n error.write(record, str(e))"
+ }, {
+ "name" : "destroyScript",
+ "value" : ""
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Parse Key-Value Pairs",
+ "xPos" : 720,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldMerger_01OutputLane15059661333060" ],
+ "outputLanes" : [ "JythonEvaluator_01OutputLane15059662339560" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ {
+ "fieldToSet" : "/event_time",
+ "expression" : "${str:regExCapture(record:value('/WhenOccurred'), \"\\\\/Date\\\\((.*)\\\\)\\\\/\", 1)}"
+ }, {
+ "fieldToSet" : "/msg",
+ "expression" : "${record:value('/EventMessage')}"
+ }, {
+ "fieldToSet" : "/user_name",
+ "expression" : "${record:value('/NormalizedUser')}"
+ }, {
+ "fieldToSet" : "/src_ip4",
+ "expression" : "${record:value('/FromIPAddress')}"
+ }, {
+ "fieldToSet" : "/user_id",
+ "expression" : "${record:value('/UserGuid')}"
+ }, {
+ "fieldToSet" : "/dst_ip4",
+ "expression" : "${record:value('/RequestHostName')}"
+ }, {
+ "fieldToSet" : "/additional_attrs",
+ "expression" : "${emptyMap()}"
+ }, {
+ "fieldToSet" : "/additional_attrs/directory_service_name",
+ "expression" : "${record:value('/DirectoryServiceName')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/auth_method",
+ "expression" : "${record:value('/AuthMethod')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/object_name",
+ "expression" : "${record:value('/ObjectName')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/object_type",
+ "expression" : "${record:value('/ObjectType')}"
+ }, {
+ "fieldToSet" : "/severity",
+ "expression" : "${record:value('/Level')}"
+ }, {
+ "fieldToSet" : "/code",
+ "expression" : "${record:value('/MfaResult')}"
+ } ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ {
+ "attributeToSet" : "p_dvc_vendor",
+ "headerAttributeExpression" : "${record:value('/product')}"
+ }, {
+ "attributeToSet" : "p_dvc_type",
+ "headerAttributeExpression" : "${record:value('/product')}"
+ }, {
+ "attributeToSet" : "p_dt",
+ "headerAttributeExpression" : "${time:extractStringFromDate(time:now(),'YYYY-MM-dd')}"
+ } ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Expression Evaluator 1",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "JythonEvaluator_01OutputLane15059662339560" ],
+ "outputLanes" : [ "ExpressionEvaluator_01OutputLane15024032304190" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldRemover_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldfilter_FieldFilterDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "filterOperation",
+ "value" : "KEEP"
+ }, {
+ "name" : "fields",
+ "value" : [ "/severity", "/raw", "/product", "/name", "/msg", "/event_time", "/dst_ip4", "/code", "/category", "/additional_attrs", "/additional_attrs/auth_method", "/additional_attrs/directory_service_name", "/additional_attrs/object_name", "/user_name", "/src_ip4", "/user_id", "/additional_attrs/object_type" ]
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Remove Extra Fields",
+ "xPos" : 1160,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_01OutputLane15024032304190" ],
+ "outputLanes" : [ "FieldRemover_01OutputLane15030862999220" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HadoopFS_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hdfs_HdfsDTarget",
+ "stageVersion" : "4",
+ "configuration" : [ {
+ "name" : "hdfsTargetConfigBean.hdfsUri",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsUser",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsKerberos",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfDir",
+ "value" : "hadoop-conf"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.uniquePrefix",
+ "value" : "sdc-${sdc:id()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.fileNameSuffix",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplateInHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplate",
+ "value" : "${ODM_EVENTS_LOCATION}/p_dvc_vendor=${record:attribute(\"p_dvc_vendor\")}/p_dvc_type=${record:attribute(\"p_dvc_type\")}/p_dt=${record:attribute(\"p_dt\")}"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeDriver",
+ "value" : "${time:now()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.maxRecordsPerFile",
+ "value" : 0
+ }, {
+ "name" : "hdfsTargetConfigBean.maxFileSize",
+ "value" : 256
+ }, {
+ "name" : "hdfsTargetConfigBean.idleTimeout",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "hdfsTargetConfigBean.otherCompression",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.fileType",
+ "value" : "TEXT"
+ }, {
+ "name" : "hdfsTargetConfigBean.keyEl",
+ "value" : "${uuid()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.seqFileCompressionType",
+ "value" : "BLOCK"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsLimit",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.rollIfHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.rollHeaderName",
+ "value" : "roll"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsAction",
+ "value" : "SEND_TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsDirPathTemplate",
+ "value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataFormat",
+ "value" : "AVRO"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsPermissionCheck",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.permissionEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.skipOldTempFileRecovery",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLines",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLinesString",
+ "value" : " "
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.jsonMode",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldPath",
+ "value" : "/text"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textRecordSeparator",
+ "value" : "\\n"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldMissingAction",
+ "value" : "ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textEmptyLineIfNull",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchemaSource",
+ "value" : "INLINE"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchema",
+ "value" : "{\n\n \"namespace\":\"org.apache.spot\",\n \"name\":\"event\",\n \"type\": \"record\",\n \"fields\": [\n {\"name\":\"event_time\",\"type\":[\"null\",\"long\"],\"doc\":\"timestamp of event (UTC)\", \"default\": null},\n {\"name\":\"duration\", \"type\":[\"null\",\"float\"],\"doc\":\"Time duration (milliseconds)\", \"default\": null},\n {\"name\":\"event_id\", \"type\":[\"null\",\"string\"],\"doc\":\"Unique identifier for event\", \"default\": null},\n {\"name\":\"name\", \"type\":[\"null\",\"string\"],\"doc\":\"Name of event\", \"default\": null},\n {\"name\":\"org\", \"type\":[\"null\",\"string\"],\"doc\":\"Organization\", \"default\": null},\n {\"name\":\"type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type information\", \"default\": null},\n {\"name\":\"n_proto\", \"type\":[\"null\",\"string\"],\"doc\":\"Network protocol of event\", \"default\": null},\n {\"name\":\"a_proto\"
, \"type\":[\"null\",\"string\"],\"doc\":\"Application protocol of event\", \"default\": null},\n {\"name\":\"msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Message (details of action taken on object)\", \"default\": null},\n {\"name\":\"mac\", \"type\":[\"null\",\"string\"],\"doc\":\"MAC address\", \"default\": null},\n {\"name\":\"severity\", \"type\":[\"null\",\"string\"],\"doc\":\"Severity of event\", \"default\": null},\n {\"name\":\"raw\", \"type\":[\"null\",\"string\"],\"doc\":\"Raw text message of entire event\", \"default\": null},\n {\"name\":\"risk\", \"type\":[\"null\",\"float\"],\"doc\":\"Risk score\", \"default\": null},\n {\"name\":\"code\", \"type\":[\"null\",\"string\"],\"doc\":\"Response or error code\", \"default\": null},\n {\"name\":\"category\", \"type\":[\"null\",\"string\"],\"doc\":\"Event category\", \"default\": null},\n {\"name\":\"query\", \"type\":[\"null\",\"string\"],\"doc\":\"Que
ry (DNS query, URI query, SQL query, etc.)\", \"default\": null},\n {\"name\":\"service\", \"type\":[\"null\",\"string\"],\"doc\":\"(i.e. service name, type of service)\", \"default\": null},\n {\"name\":\"state\", \"type\":[\"null\",\"string\"],\"doc\":\"State of object\", \"default\": null},\n {\"name\":\"in_bytes\", \"type\":[\"null\",\"int\"],\"doc\":\"Bytes in\", \"default\": null},\n {\"name\":\"out_bytes\", \"type\":[\"null\",\"int\"],\"doc\":\"Bytes out\", \"default\": null},\n {\"name\":\"xref\", \"type\":[\"null\",\"string\"],\"doc\":\"External reference to public description\", \"default\": null},\n {\"name\":\"version\", \"type\":[\"null\",\"string\"],\"doc\":\"Version\", \"default\": null},\n {\"name\":\"additional_attrs\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\", \"string\"]}],\"default\":null, \"doc\":\"Additional attributes of the event\"},\n {\"name\":\"dvc_time\", \"type\":[\"null\
",\"long\"],\"doc\":\"UTC timestamp from device where event/alert originates or is received\", \"default\": null},\n {\"name\":\"dvc_ip4\", \"type\":[\"null\",\"string\"],\"doc\":\"IP address of device\", \"default\": null},\n {\"name\":\"dvc_ip6\", \"type\":[\"null\",\"string\"],\"doc\":\"IP address of device\", \"default\": null},\n {\"name\":\"dvc_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Hostname of device\", \"default\": null},\n {\"name\":\"dvc_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Device type that generated the log\", \"default\": null},\n {\"name\":\"dvc_vendor\", \"type\":[\"null\",\"string\"],\"doc\":\"Vendor\", \"default\": null},\n {\"name\":\"dvc_fwd_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Forwarded from device\", \"default\": null},\n {\"name\":\"dvc_fwd_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Forwarded from device\", \"default\": null},\n {\"name\":\"dvc_version\", \"type\":[\"null\"
,\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"src_ip4\", \"type\":[\"null\",\"string\"],\"doc\":\"Source ip address of event\", \"default\": null},\n {\"name\":\"src_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Source ip address of event\", \"default\": null},\n {\"name\":\"src_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Source FQDN of event\", \"default\": null},\n {\"name\":\"src_domain\", \"type\":[\"null\",\"string\"],\"doc\":\"Domain name of source address\", \"default\": null},\n {\"name\":\"src_port\", \"type\":[\"null\",\"int\"],\"doc\":\"Source port of event\", \"default\": null},\n {\"name\":\"src_country_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country code\", \"default\": null},\n {\"name\":\"src_country_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country name\", \"default\": null},\n {\"name\":\"src_region\", \"type\":[\"null\",\"string\"],\"doc\":\"Source region\"
, \"default\": null},\n {\"name\":\"src_city\", \"type\":[\"null\",\"string\"],\"doc\":\"Source city\", \"default\": null},\n {\"name\":\"src_lat\", \"type\":[\"null\",\"int\"],\"doc\":\"Source latitude\", \"default\": null},\n {\"name\":\"src_long\", \"type\":[\"null\",\"int\"],\"doc\":\"Source longitude\", \"default\": null},\n {\"name\":\"dst_ip4\", \"type\":[\"null\",\"string\"],\"doc\":\"Destination ip address of event\", \"default\": null},\n {\"name\":\"dst_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Destination ip address of event\", \"default\": null},\n {\"name\":\"dst_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Destination FQDN of event\", \"default\": null},\n {\"name\":\"dst_domain\", \"type\":[\"null\",\"string\"],\"doc\":\"Domain name of destination address\", \"default\": null},\n {\"name\":\"dst_port\", \"type\":[\"null\",\"int\"],\"doc\":\"Destination port of event\", \"default\": null},\n {\"nam
e\":\"dst_country_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country code\", \"default\": null},\n {\"name\":\"dst_country_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country name\", \"default\": null},\n {\"name\":\"dst_region\", \"type\":[\"null\",\"string\"],\"doc\":\"Source region\", \"default\": null},\n {\"name\":\"dst_city\", \"type\":[\"null\",\"string\"],\"doc\":\"Source city\", \"default\": null},\n {\"name\":\"dst_lat\", \"type\":[\"null\",\"int\"],\"doc\":\"Source latitude\", \"default\": null},\n {\"name\":\"dst_long\", \"type\":[\"null\",\"int\"],\"doc\":\"Source longitude\", \"default\": null},\n {\"name\":\"src_asn\", \"type\":[\"null\",\"int\"],\"doc\":\"Autonomous system number\", \"default\": null},\n {\"name\":\"dst_asn\", \"type\":[\"null\",\"int\"],\"doc\":\"Autonomous system number\", \"default\": null},\n {\"name\":\"net_direction\", \"type\":[\"null\",\"string\"],\"doc\":\"Direc
tion\", \"default\": null},\n {\"name\":\"net_flags\", \"type\":[\"null\",\"string\"],\"doc\":\"TCP flags\", \"default\": null},\n {\"name\":\"file_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Filename from event\", \"default\": null},\n {\"name\":\"file_path\", \"type\":[\"null\",\"string\"],\"doc\":\"File path\", \"default\": null},\n {\"name\":\"file_atime\", \"type\":[\"null\",\"long\"],\"doc\":\"Timestamp (UTC) of file access\", \"default\": null},\n {\"name\":\"file_acls\", \"type\":[\"null\",\"string\"],\"doc\":\"File permissions\", \"default\": null},\n {\"name\":\"file_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of file\", \"default\": null},\n {\"name\":\"file_size\", \"type\":[\"null\",\"int\"],\"doc\":\"Size of file in bytes\", \"default\": null},\n {\"name\":\"file_desc\", \"type\":[\"null\",\"string\"],\"doc\":\"Description of file\", \"default\": null},\n {\"name\":\"file_hash\", \"type\":[\"
null\",\"string\"],\"doc\":\"Hash of file\", \"default\": null},\n {\"name\":\"file_hash_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of hash\", \"default\": null},\n {\"name\":\"end_object\", \"type\":[\"null\",\"string\"],\"doc\":\"File/Process/ Registry\", \"default\": null},\n {\"name\":\"end_action\", \"type\":[\"null\",\"string\"],\"doc\":\"Action taken on object (open/delete/ edit)\", \"default\": null},\n {\"name\":\"end_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Message (details of action taken on object)\", \"default\": null},\n {\"name\":\"end_app\", \"type\":[\"null\",\"string\"],\"doc\":\"Application\", \"default\": null},\n {\"name\":\"end_location\", \"type\":[\"null\",\"string\"],\"doc\":\"Location\", \"default\": null},\n {\"name\":\"end_proc\", \"type\":[\"null\",\"string\"],\"doc\":\"Process\", \"default\": null},\n {\"name\":\"user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from
event\", \"default\": null},\n {\"name\":\"src_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"dst_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"user_email\", \"type\":[\"null\",\"string\"],\"doc\":\"Email address\", \"default\": null},\n {\"name\":\"user_id\", \"type\":[\"null\",\"string\"],\"doc\":\"userid\", \"default\": null},\n {\"name\":\"user_loc\", \"type\":[\"null\",\"string\"],\"doc\":\"location\", \"default\": null},\n {\"name\":\"user_desc\", \"type\":[\"null\",\"string\"],\"doc\":\"Description of user\", \"default\": null},\n {\"name\":\"dns_class\", \"type\":[\"null\",\"string\"],\"doc\":\"DNS class\", \"default\": null},\n {\"name\":\"dns_len\", \"type\":[\"null\",\"int\"],\"doc\":\"DNS frame length\", \"default\": null},\n {\"name\":\"dns_query\", \"type\":[\"null\",\"string\"],\
"doc\":\"Requested DNS query\", \"default\": null},\n {\"name\":\"dns_response_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Response code\", \"default\": null},\n {\"name\":\"dns_answers\", \"type\":[\"null\",\"string\"],\"doc\":\"Response to DNS Query\", \"default\": null},\n {\"name\":\"dns_type\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"prx_category\", \"type\":[\"null\",\"string\"],\"doc\":\"Event category\", \"default\": null},\n {\"name\":\"prx_browser\", \"type\":[\"null\",\"string\"],\"doc\":\"Web browser\", \"default\": null},\n {\"name\":\"prx_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Error or response code\", \"default\": null},\n {\"name\":\"prx_referrer\", \"type\":[\"null\",\"string\"],\"doc\":\"Referrer\", \"default\": null},\n {\"name\":\"prx_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Requested URI\", \"default\": null},\n {\"name\":\"prx_filter_rule
\", \"type\":[\"null\",\"string\"],\"doc\":\"Applied filter or rule\", \"default\": null},\n {\"name\":\"prx_filter_result\", \"type\":[\"null\",\"string\"],\"doc\":\"Result of applied filter or rule\", \"default\": null},\n {\"name\":\"prx_query\", \"type\":[\"null\",\"string\"],\"doc\":\"URI query\", \"default\": null},\n {\"name\":\"prx_action\", \"type\":[\"null\",\"string\"],\"doc\":\"Action taken on object\", \"default\": null},\n {\"name\":\"prx_method\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP method\", \"default\": null},\n {\"name\":\"prx_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of request\", \"default\": null},\n {\"name\":\"http_request_method\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP method\", \"default\": null},\n {\"name\":\"http_request_uri\", \"type\":[\"null\",\"string\"],\"doc\":\"Requested URI\", \"default\": null},\n {\"name\":\"http_request_body_len\", \"type\":[\"null\",\"int\"],
\"doc\":\"Length of request body\", \"default\": null},\n {\"name\":\"http_request_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"http_request_password\", \"type\":[\"null\",\"string\"],\"doc\":\"Password from event\", \"default\": null},\n {\"name\":\"http_request_proxied\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"http_request_headers\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP request headers\", \"default\": null},\n {\"name\":\"http_response_status_code\", \"type\":[\"null\",\"int\"],\"doc\":\"HTTP response status code\", \"default\": null},\n {\"name\":\"http_response_status_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP response status message\", \"default\": null},\n {\"name\":\"http_response_body_len\", \"type\":[\"null\",\"string\"],\"doc\":\"Length of response body\", \"default\": null},\n {\"name\":\"ht
tp_response_info_code\", \"type\":[\"null\",\"int\"],\"doc\":\"HTTP response info code\", \"default\": null},\n {\"name\":\"http_response_info_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP response info message\", \"default\": null},\n {\"name\":\"http_response_resp_fuids\", \"type\":[\"null\",\"string\"],\"doc\":\"Response FUIDS\", \"default\": null},\n {\"name\":\"http_response_mime_types\", \"type\":[\"null\",\"string\"],\"doc\":\"Mime types\", \"default\": null},\n {\"name\":\"http_response_headers\", \"type\":[\"null\",\"string\"],\"doc\":\"Response headers\", \"default\": null},\n {\"name\":\"smtp_trans_depth\", \"type\":[\"null\",\"int\"],\"doc\":\"Depth of email into SMTP exchange\", \"default\": null},\n {\"name\":\"smtp_headers_helo\", \"type\":[\"null\",\"string\"],\"doc\":\"Helo header\", \"default\": null},\n {\"name\":\"smtp_headers_mailfrom\", \"type\":[\"null\",\"string\"],\"doc\":\"Mailfrom header\", \"default\
": null},\n {\"name\":\"smtp_headers_rcptto\", \"type\":[\"null\",\"string\"],\"doc\":\"Rcptto header\", \"default\": null},\n {\"name\":\"smtp_headers_date\", \"type\":[\"null\",\"string\"],\"doc\":\"Header date\", \"default\": null},\n {\"name\":\"smtp_headers_from\", \"type\":[\"null\",\"string\"],\"doc\":\"From header\", \"default\": null},\n {\"name\":\"smtp_headers_to\", \"type\":[\"null\",\"string\"],\"doc\":\"To header\", \"default\": null},\n {\"name\":\"smtp_headers_reply_to\", \"type\":[\"null\",\"string\"],\"doc\":\"Reply to header\", \"default\": null},\n {\"name\":\"smtp_headers_msg_id\", \"type\":[\"null\",\"string\"],\"doc\":\"Message ID\", \"default\": null},\n {\"name\":\"smtp_headers_in_reply_to\", \"type\":[\"null\",\"string\"],\"doc\":\"In reply to header\", \"default\": null},\n {\"name\":\"smpt_headers_subject\", \"type\":[\"null\",\"string\"],\"doc\":\"Subject\", \"default\": null},\n {\"name\":\"
smtp_headers_x_originating_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Originating IP address\", \"default\": null},\n {\"name\":\"smtp_headers_first_received\", \"type\":[\"null\",\"string\"],\"doc\":\"First to receive message\", \"default\": null},\n {\"name\":\"smtp_headers_second_received\", \"type\":[\"null\",\"string\"],\"doc\":\"Second to receive message\", \"default\": null},\n {\"name\":\"smtp_last_reply\", \"type\":[\"null\",\"string\"],\"doc\":\"Last reply in message chain\", \"default\": null},\n {\"name\":\"smtp_path\", \"type\":[\"null\",\"string\"],\"doc\":\"Path of message\", \"default\": null},\n {\"name\":\"smtp_user_agent\", \"type\":[\"null\",\"string\"],\"doc\":\"User agent\", \"default\": null},\n {\"name\":\"smtp_tls\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Indication of TLS use\", \"default\": null},\n {\"name\":\"smtp_is_webmail\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Indication of webmail\", \"defa
ult\": null},\n {\"name\":\"ftp_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Username\", \"default\": null},\n {\"name\":\"ftp_password\", \"type\":[\"null\",\"string\"],\"doc\":\"Password\", \"default\": null},\n {\"name\":\"ftp_command\", \"type\":[\"null\",\"string\"],\"doc\":\"FTP command\", \"default\": null},\n {\"name\":\"ftp_arg\", \"type\":[\"null\",\"string\"],\"doc\":\"Argument\", \"default\": null},\n {\"name\":\"ftp_mime_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Mime type\", \"default\": null},\n {\"name\":\"ftp_file_size\", \"type\":[\"null\",\"int\"],\"doc\":\"File size\", \"default\": null},\n {\"name\":\"ftp_reply_code\", \"type\":[\"null\",\"int\"],\"doc\":\"Reply code\", \"default\": null},\n {\"name\":\"ftp_reply_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Reply message\", \"default\": null},\n {\"name\":\"ftp_data_channel_passive\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Passiv
e data channel?\", \"default\": null},\n {\"name\":\"ftp_data_channel_rsp_p\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_cwd\", \"type\":[\"null\",\"string\"],\"doc\":\"Current working directory\", \"default\": null},\n {\"name\":\"ftp_cmdarg_ts\", \"type\":[\"null\",\"float\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_cmdarg_cmd\", \"type\":[\"null\",\"string\"],\"doc\":\"Command\", \"default\": null},\n {\"name\":\"ftp_cmdarg_arg\", \"type\":[\"null\",\"string\"],\"doc\":\"Command argument\", \"default\": null},\n {\"name\":\"ftp_cmdarg_seq\", \"type\":[\"null\",\"int\"],\"doc\":\"Sequence\", \"default\": null},\n {\"name\":\"ftp_pending_commands\", \"type\":[\"null\",\"string\"],\"doc\":\"Pending commands\", \"default\": null},\n {\"name\":\"ftp_is_passive\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Passive mode enabled\", \"default\": null},\n {\"name\":\"ftp_
fuid\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_last_auth_requested\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_community\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_bulk_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_responses\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_set_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_display_string\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_up_since\", \"type\":
[\"null\",\"float\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_cipher\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_curve\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_server_name\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_resumed\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_next_protocol\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_established\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_cert_chain_fuids\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_client_cert_chain_fuids\", \"type\":[\"null\",\"string\"],\"
doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_auth_success\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_client\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_server\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_cipher_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_mac_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_compression_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_key_exchange_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_host_key_algorithm\", \"type\":[\"null\",\"string\"],\"do
c\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_assigned_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_mac\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_lease_time\", \"type\":[\"null\",\"double\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_user\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_nickname\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_command\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_value\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_additional_data\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_in_packets\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\"
:\"flow_out_packets\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_conn_state\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_history\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_src_dscp\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_dst_dscp\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_input\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_output\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_id\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_title\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_type\", \"type\":[\"null\",\"string\"],\"doc\":\
"TBD\", \"default\": null},\n {\"name\":\"vuln_status\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_severity\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null}\n ],\n \"doc\": \"A view schema for storing Apache Spot Event data.\"\n }"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.registerSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subjectToRegister",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroCompression",
+ "value" : "NULL"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.binaryFieldPath",
+ "value" : "/"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.fileNameEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.wholeFileExistsAction",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.includeChecksumInTheEvents",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.checksumAlgorithm",
+ "value" : "MD5"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlPrettyPrint",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlValidateSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlSchema",
+ "value" : null
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Hadoop FS 1",
+ "xPos" : 1381.5806884765625,
+ "yPos" : 51.58061981201172,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ "FieldRemover_01OutputLane15030862999220" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ "HadoopFS_01_EventLane" ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_02",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ {
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -4), \".*=(.*)\", 1)}",
+ "attributeToSet" : "p_dvc_vendor"
+ }, {
+ "attributeToSet" : "p_dvc_type",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -3), \".*=(.*)\", 1)}"
+ }, {
+ "attributeToSet" : "p_dt",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -2), \".*=(.*)\", 1)}"
+ } ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Extract Partition Values",
+ "xPos" : 1600,
+ "yPos" : 200,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "HadoopFS_01_EventLane" ],
+ "outputLanes" : [ "ExpressionEvaluator_02OutputLane15059686811520" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HiveQuery_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "config.hiveConfigBean.hiveJDBCUrl",
+ "value" : "${HIVE_URL}"
+ }, {
+ "name" : "config.hiveConfigBean.hiveJDBCDriver",
+ "value" : "org.apache.hive.jdbc.HiveDriver"
+ }, {
+ "name" : "config.hiveConfigBean.confDir",
+ "value" : "hive-conf"
+ }, {
+ "name" : "config.hiveConfigBean.additionalConfigProperties",
+ "value" : [ ]
+ }, {
+ "name" : "config.queries",
+ "value" : [ "ALTER TABLE ${ODM_EVENTS_TABLE_NAME} ADD IF NOT EXISTS PARTITION (p_dvc_type='${record:attribute(\"p_dvc_type\")}', p_dvc_vendor='${record:attribute(\"p_dvc_vendor\")}', p_dt='${record:attribute(\"p_dt\")}')" ]
+ }, {
+ "name" : "config.stopOnQueryFailure",
+ "value" : true
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Add Partition to Avro Table",
+ "xPos" : 1820,
+ "yPos" : 200,
+ "stageType" : "EXECUTOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_02OutputLane15059686811520" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "errorStage" : {
+ "instanceName" : "Discard_ErrorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Error Records - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "info" : {
+ "pipelineId" : "CentrifyUserEventTCPf5cff562-2e3b-46c7-9dd1-295df3c10b74",
+ "title" : "ODMCentrifyIdentityPlatformEventTCP",
+ "description" : "",
+ "created" : 1505970571031,
+ "lastModified" : 1505971604217,
+ "creator" : "natty@dpmfield",
+ "lastModifier" : "natty@dpmfield",
+ "lastRev" : "0",
+ "uuid" : "1fac1baa-3ae7-4aa0-a289-80bc39552100",
+ "valid" : true,
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "name" : "CentrifyUserEventTCPf5cff562-2e3b-46c7-9dd1-295df3c10b74",
+ "sdcVersion" : "2.7.1.0",
+ "sdcId" : "f0ff6a12-61b1-44a6-bc5f-0d7e67d7d482"
+ },
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "statsAggregatorStage" : {
+ "instanceName" : "WritetoDPMdirectly_StatsAggregatorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stats Aggregator - Write to DPM directly",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "startEventStages" : [ {
+ "instanceName" : "Discard_StartEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Start Event - Discard",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "stopEventStages" : [ {
+ "instanceName" : "Discard_StopEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stop Event - Discard",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "valid" : true,
+ "issues" : {
+ "issueCount" : 0,
+ "stageIssues" : { },
+ "pipelineIssues" : [ ]
+ },
+ "previewable" : true
+ },
+ "pipelineRules" : {
+ "schemaVersion" : 3,
+ "version" : 2,
+ "metricsRuleDefinitions" : [ {
+ "id" : "badRecordsAlertID",
+ "alertText" : "High incidence of Error Records",
+ "metricId" : "pipeline.batchErrorRecords.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "stageErrorAlertID",
+ "alertText" : "High incidence of Stage Errors",
+ "metricId" : "pipeline.batchErrorMessages.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "idleGaugeID",
+ "alertText" : "Pipeline is Idle",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
+ "condition" : "${time:now() - value() > 120000}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "batchTimeAlertID",
+ "alertText" : "Batch taking more time to process",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "CURRENT_BATCH_AGE",
+ "condition" : "${value() > 200}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "memoryLimitAlertID",
+ "alertText" : "Memory limit for pipeline exceeded",
+ "metricId" : "pipeline.memoryConsumed.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > (jvm:maxMemoryMB() * 0.65)}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ } ],
+ "dataRuleDefinitions" : [ ],
+ "driftRuleDefinitions" : [ ],
+ "uuid" : "3cab303e-a7d9-4c7c-b9b2-f4c822fd36a5",
+ "configuration" : [ {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ } ],
+ "ruleIssues" : [ ],
+ "configIssues" : [ ]
+ },
+ "libraryDefinitions" : null
+}
\ No newline at end of file
[5/7] incubator-spot git commit: SPOT-227. [Ingest] StreamSets
pipeline configs for Qualys ingest into ODM
Posted by na...@apache.org.
SPOT-227. [Ingest] StreamSets pipeline configs for Qualys ingest into ODM
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/b6bba644
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/b6bba644
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/b6bba644
Branch: refs/heads/SPOT-181_ODM
Commit: b6bba644e93681ae742953581176e42b80b732c4
Parents: ed388f7
Author: Jon Natkins <na...@streamsets.com>
Authored: Wed Sep 20 22:47:06 2017 -0700
Committer: Jon Natkins <na...@streamsets.com>
Committed: Thu Sep 21 13:43:25 2017 -0700
----------------------------------------------------------------------
.../qualys/ODMQualysVulnerabilityContext.json | 1276 ++++++++++++++++++
.../qualys/ODMQualysVulnerabilityEvents.json | 1245 +++++++++++++++++
2 files changed, 2521 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b6bba644/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityContext.json
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityContext.json b/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityContext.json
new file mode 100644
index 0000000..f17088b
--- /dev/null
+++ b/spot-ingest/streamsets/qualys/ODMQualysVulnerabilityContext.json
@@ -0,0 +1,1276 @@
+{
+ "pipelineConfig" : {
+ "schemaVersion" : 4,
+ "version" : 7,
+ "pipelineId" : "QualysVulnerabilityContextTeste17011d3-a07e-43c8-a85a-0c403f9db901",
+ "title" : "ODMQualysVulnerabilityContext",
+ "description" : "",
+ "uuid" : "97e5f9c0-3e2a-4686-b3d1-17eaf3e1d79b",
+ "configuration" : [ {
+ "name" : "executionMode",
+ "value" : "STANDALONE"
+ }, {
+ "name" : "deliveryGuarantee",
+ "value" : "AT_LEAST_ONCE"
+ }, {
+ "name" : "shouldRetry",
+ "value" : true
+ }, {
+ "name" : "retryAttempts",
+ "value" : -1
+ }, {
+ "name" : "memoryLimit",
+ "value" : "${jvm:maxMemoryMB() * 0.65}"
+ }, {
+ "name" : "memoryLimitExceeded",
+ "value" : "STOP_PIPELINE"
+ }, {
+ "name" : "notifyOnStates",
+ "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "constants",
+ "value" : [ {
+ "key" : "QUALYS_API_URL",
+ "value" : ""
+ }, {
+ "key" : "ODM_VULN_CTX_LOCATION",
+ "value" : ""
+ }, {
+ "key" : "ODM_VULN_CTX_PARQUET_LOCATION",
+ "value" : ""
+ } ]
+ }, {
+ "name" : "badRecordsHandling",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "clusterSlaveMemory",
+ "value" : 1024
+ }, {
+ "name" : "clusterSlaveJavaOpts",
+ "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
+ }, {
+ "name" : "clusterLauncherEnv",
+ "value" : [ ]
+ }, {
+ "name" : "mesosDispatcherURL",
+ "value" : null
+ }, {
+ "name" : "hdfsS3ConfDir",
+ "value" : null
+ }, {
+ "name" : "rateLimit",
+ "value" : 0
+ }, {
+ "name" : "maxRunners",
+ "value" : 0
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "statsAggregatorStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
+ }, {
+ "name" : "workerCount",
+ "value" : 0
+ }, {
+ "name" : "startEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "stopEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "errorRecordPolicy",
+ "value" : "ORIGINAL_RECORD"
+ }, {
+ "name" : "sparkConfigs",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "previewConfig" : {
+ "previewSource" : "CONFIGURED_SOURCE",
+ "batchSize" : 10,
+ "timeout" : 10000,
+ "writeToDestinations" : true,
+ "showHeader" : true,
+ "showFieldType" : true,
+ "rememberMe" : false
+ }
+ },
+ "stages" : [ {
+ "instanceName" : "HTTPClient_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_origin_http_HttpClientDSource",
+ "stageVersion" : "13",
+ "configuration" : [ {
+ "name" : "conf.basic.maxBatchSize",
+ "value" : 1000
+ }, {
+ "name" : "conf.basic.maxWaitTime",
+ "value" : 2000
+ }, {
+ "name" : "conf.dataFormatConfig.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.dataFormatConfig.filePatternInArchive",
+ "value" : "*"
+ }, {
+ "name" : "conf.dataFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.dataFormatConfig.removeCtrlChars",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.textMaxLineLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.useCustomDelimiter",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customDelimiter",
+ "value" : "\\r\\n"
+ }, {
+ "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.jsonMaxObjectLen",
+ "value" : 4096
+ }, {
+ "name" : "conf.dataFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "conf.dataFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "conf.dataFormatConfig.csvMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.csvAllowExtraColumns",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
+ "value" : "_extra_"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "conf.dataFormatConfig.csvEnableComments",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvCommentMarker",
+ "value" : "#"
+ }, {
+ "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.csvRecordType",
+ "value" : "LIST_MAP"
+ }, {
+ "name" : "conf.dataFormatConfig.csvSkipStartLines",
+ "value" : 0
+ }, {
+ "name" : "conf.dataFormatConfig.parseNull",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.nullConstant",
+ "value" : "\\\\N"
+ }, {
+ "name" : "conf.dataFormatConfig.xmlRecordElement",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xPathNamespaceContext",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.outputFieldAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xmlMaxObjectLen",
+ "value" : 4096000
+ }, {
+ "name" : "conf.dataFormatConfig.logMode",
+ "value" : "COMMON_LOG_FORMAT"
+ }, {
+ "name" : "conf.dataFormatConfig.logMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.retainOriginalLine",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customLogFormat",
+ "value" : "%h %l %u %t \"%r\" %>s %b"
+ }, {
+ "name" : "conf.dataFormatConfig.regex",
+ "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
+ }, {
+ "name" : "conf.dataFormatConfig.fieldPathsToGroupName",
+ "value" : [ {
+ "fieldPath" : "/",
+ "group" : 1
+ } ]
+ }, {
+ "name" : "conf.dataFormatConfig.grokPatternDefinition",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.grokPattern",
+ "value" : "%{COMMONAPACHELOG}"
+ }, {
+ "name" : "conf.dataFormatConfig.onParseError",
+ "value" : "ERROR"
+ }, {
+ "name" : "conf.dataFormatConfig.maxStackTraceLines",
+ "value" : 50
+ }, {
+ "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.log4jCustomLogFormat",
+ "value" : "%r [%t] %-5p %c %x - %m%n"
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchemaSource",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchema",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "conf.dataFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.isDelimited",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.binaryMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.datagramMode",
+ "value" : "SYSLOG"
+ }, {
+ "name" : "conf.dataFormatConfig.typesDbPath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.convertTime",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.excludeInterval",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.authFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
+ "value" : 8192
+ }, {
+ "name" : "conf.dataFormatConfig.rateLimit",
+ "value" : "-1"
+ }, {
+ "name" : "conf.dataFormatConfig.verifyChecksum",
+ "value" : false
+ }, {
+ "name" : "conf.resourceUrl",
+ "value" : "${QUALYS_API_URL}"
+ }, {
+ "name" : "conf.headers",
+ "value" : [ ]
+ }, {
+ "name" : "conf.httpMethod",
+ "value" : "GET"
+ }, {
+ "name" : "conf.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "conf.requestBody",
+ "value" : null
+ }, {
+ "name" : "conf.defaultRequestContentType",
+ "value" : "application/json"
+ }, {
+ "name" : "conf.client.transferEncoding",
+ "value" : "CHUNKED"
+ }, {
+ "name" : "conf.client.httpCompression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.connectTimeoutMillis",
+ "value" : 0
+ }, {
+ "name" : "conf.client.readTimeoutMillis",
+ "value" : 0
+ }, {
+ "name" : "conf.client.authType",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.useOAuth2",
+ "value" : false
+ }, {
+ "name" : "conf.client.oauth.consumerKey",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.consumerSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.token",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth.tokenSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.credentialsGrantType",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.tokenUrl",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.clientId",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.clientSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.resourceOwnerClientId",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.resourceOwnerClientSecret",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.algorithm",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.client.oauth2.key",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.jwtClaims",
+ "value" : null
+ }, {
+ "name" : "conf.client.oauth2.transferEncoding",
+ "value" : "BUFFERED"
+ }, {
+ "name" : "conf.client.oauth2.additionalValues",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.basicAuth.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.basicAuth.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.useProxy",
+ "value" : false
+ }, {
+ "name" : "conf.client.proxy.uri",
+ "value" : null
+ }, {
+ "name" : "conf.client.proxy.username",
+ "value" : null
+ }, {
+ "name" : "conf.client.proxy.password",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.tlsEnabled",
+ "value" : false
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.client.tlsConfig.keyStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.keyStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreType",
+ "value" : "JKS"
+ }, {
+ "name" : "conf.client.tlsConfig.trustStorePassword",
+ "value" : null
+ }, {
+ "name" : "conf.client.tlsConfig.trustStoreAlgorithm",
+ "value" : "SunX509"
+ }, {
+ "name" : "conf.client.tlsConfig.useDefaultProtocols",
+ "value" : true
+ }, {
+ "name" : "conf.client.tlsConfig.protocols",
+ "value" : [ ]
+ }, {
+ "name" : "conf.client.tlsConfig.useDefaultCiperSuites",
+ "value" : true
+ }, {
+ "name" : "conf.client.tlsConfig.cipherSuites",
+ "value" : [ ]
+ }, {
+ "name" : "conf.httpMode",
+ "value" : "POLLING"
+ }, {
+ "name" : "conf.pollingInterval",
+ "value" : 5000
+ }, {
+ "name" : "conf.dataFormat",
+ "value" : "XML"
+ }, {
+ "name" : "conf.responseStatusActionConfigs",
+ "value" : [ {
+ "statusCode" : 500,
+ "action" : "RETRY_EXPONENTIAL_BACKOFF",
+ "backoffInterval" : 1000,
+ "maxNumRetries" : 10
+ } ]
+ }, {
+ "name" : "conf.responseTimeoutActionConfig.action",
+ "value" : "RETRY_IMMEDIATELY"
+ }, {
+ "name" : "conf.responseTimeoutActionConfig.backoffInterval",
+ "value" : 1000
+ }, {
+ "name" : "conf.responseTimeoutActionConfig.maxNumRetries",
+ "value" : 10
+ }, {
+ "name" : "conf.pagination.mode",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.pagination.startAt",
+ "value" : null
+ }, {
+ "name" : "conf.pagination.resultFieldPath",
+ "value" : null
+ }, {
+ "name" : "conf.pagination.keepAllFields",
+ "value" : false
+ }, {
+ "name" : "conf.pagination.rateLimit",
+ "value" : 2000
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "HTTP Client 1",
+ "xPos" : 58.24463653564453,
+ "yPos" : 51.75536346435547,
+ "stageType" : "SOURCE"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ "HTTPClient_01OutputLane15059268799440" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldPivoter_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "listPath",
+ "value" : "/RESPONSE[0]/VULN_LIST[0]/VULN"
+ }, {
+ "name" : "copyFields",
+ "value" : true
+ }, {
+ "name" : "newPath",
+ "value" : ""
+ }, {
+ "name" : "saveOriginalFieldName",
+ "value" : false
+ }, {
+ "name" : "originalFieldNamePath",
+ "value" : null
+ }, {
+ "name" : "onStagePreConditionFailure",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Pivot Vulnerability List",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "HTTPClient_01OutputLane15059268799440" ],
+ "outputLanes" : [ "FieldPivoter_01OutputLane15024033858660" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldPivoter_02",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "listPath",
+ "value" : "/RESPONSE[0]/VULN_LIST[0]/VULN/SOFTWARE_LIST[0]/SOFTWARE"
+ }, {
+ "name" : "copyFields",
+ "value" : true
+ }, {
+ "name" : "newPath",
+ "value" : null
+ }, {
+ "name" : "saveOriginalFieldName",
+ "value" : false
+ }, {
+ "name" : "originalFieldNamePath",
+ "value" : null
+ }, {
+ "name" : "onStagePreConditionFailure",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Pivot Affected Software List",
+ "xPos" : 500,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldPivoter_01OutputLane15024033858660" ],
+ "outputLanes" : [ "FieldPivoter_02OutputLane15024040316830" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldPivoter_04",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_listpivot_ListPivotDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "listPath",
+ "value" : "/RESPONSE[0]/VULN_LIST[0]/VULN/VENDOR_REFERENCE_LIST[0]/VENDOR_REFERENCE"
+ }, {
+ "name" : "copyFields",
+ "value" : true
+ }, {
+ "name" : "newPath",
+ "value" : null
+ }, {
+ "name" : "saveOriginalFieldName",
+ "value" : false
+ }, {
+ "name" : "originalFieldNamePath",
+ "value" : null
+ }, {
+ "name" : "onStagePreConditionFailure",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Pivot Vendor Ref List",
+ "xPos" : 720,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldPivoter_02OutputLane15024040316830" ],
+ "outputLanes" : [ "FieldPivoter_04OutputLane15030890171900" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ {
+ "fieldToSet" : "/vuln_id",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/QID[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_type",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/VULN_TYPE[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_title",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/TITLE[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_severity",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/SEVERITY_LEVEL[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_description",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/DIAGNOSIS[0]/value')}, ${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/CONSEQUENCE[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_category",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/CATEGORY[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_solution",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/SOLUTION[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_created",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/PUBLISHED_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/vuln_updated",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/LAST_SERVICE_MODIFICATION_DATETIME[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs",
+ "expression" : "${emptyMap()}"
+ }, {
+ "fieldToSet" : "/additional_attrs/patchable",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/PATCHABLE[0]/value')}"
+ }, {
+ "fieldToSet" : "/additional_attrs/pci_flag",
+ "expression" : "${record:value('/RESPONSE[0]/VULN_LIST[0]/VULN/PCI_FLAG[0]/value')}"
+ } ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Map Fields to ODM",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldPivoter_04OutputLane15030890171900" ],
+ "outputLanes" : [ "ExpressionEvaluator_01OutputLane15030893927620" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "GroovyEvaluator_01",
+ "library" : "streamsets-datacollector-groovy_2_4-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_groovy_GroovyDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "processingMode",
+ "value" : "BATCH"
+ }, {
+ "name" : "initScript",
+ "value" : "/*\n * Available objects:\n * state: A Map<String, Object> that is preserved between invocations of this script.\n * Useful for caching bits of data, e.g. counters.\n *\n * log.<level>(msg, obj...): Use instead of println to send log messages to the log4j log\n * instead of stdout.\n * loglevel is any log4j level: e.g. info, warn, error, trace.\n * sdcFunctions.getFieldNull(Record, 'field path'): Receive a constant defined above \n * to check if the field is typed field with value null\n * sdcFunctions.createMap(boolean listMap): Create a map for use as a field in a record. \n * Pass true to this function to create a list map (ordered map)\n * sdcFunctions.pipelineParameters(): Map with pipeline runtime parameters.\n */\n\n// state['connection'] = new Connection().open();"
+ }, {
+ "name" : "script",
+ "value" : "for (record in records) {\n try {\n record.value['additional_attrs']['cve_list'] = \n record.value['RESPONSE'][0]['VULN_LIST'][0]['VULN']['CVE_LIST'][0]['CVE'].collect { \n it['ID'][0]['value']\n }.join(\", \")\n output.write(record)\n } catch (e) {\n // Write a record to the error pipeline\n log.error(e.toString(), e)\n error.write(record, e.toString())\n }\n}"
+ }, {
+ "name" : "destroyScript",
+ "value" : "/*\n * Available objects:\n * state: A Map<String, Object> that is preserved between invocations of this script.\n * Useful for caching bits of data, e.g. counters.\n *\n * log.<level>(msg, obj...): Use instead of println to send log messages to the log4j log\n * instead of stdout.\n * loglevel is any log4j level: e.g. info, warn, error, trace.\n * sdcFunctions.getFieldNull(Record, 'field path'): Receive a constant defined above \n * to check if the field is typed field with value null\n * sdcFunctions.createMap(boolean listMap): Create a map for use as a field in a record. \n * Pass true to this function to create a list map (ordered map)\n * sdcFunctions.createEvent(String type, int version): Creates a new event.\n * Create new empty event with standard headers.\n * sdcFunctions.toEvent(Record): Send event to eve
nt stream\n * Only events created with sdcFunctions.createEvent are supported.\n * sdcFunctions.pipelineParameters(): Map with pipeline runtime parameters.\n *\n */\n\n// state?.connection.close()"
+ }, {
+ "name" : "invokeDynamic",
+ "value" : false
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Convert CVE List to String",
+ "xPos" : 1160,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_01OutputLane15030893927620" ],
+ "outputLanes" : [ "GroovyEvaluator_01OutputLane15030909339850" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldRemover_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldfilter_FieldFilterDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "filterOperation",
+ "value" : "REMOVE"
+ }, {
+ "name" : "fields",
+ "value" : [ "/RESPONSE" ]
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Remove Extra Fields",
+ "xPos" : 1380,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "GroovyEvaluator_01OutputLane15030909339850" ],
+ "outputLanes" : [ "FieldRemover_01OutputLane15030862999220" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldTypeConverter_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldtypeconverter_FieldTypeConverterDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "convertBy",
+ "value" : "BY_FIELD"
+ }, {
+ "name" : "fieldTypeConverterConfigs",
+ "value" : [ {
+ "fields" : [ "/vuln_created", "/vuln_updated" ],
+ "targetType" : "DATETIME",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "OTHER",
+ "encoding" : "UTF-8",
+ "otherDateFormat" : "yyyy-MM-dd'T'HH:mm:ss'Z'"
+ }, {
+ "fields" : [ "/vuln_updated", "/vuln_created" ],
+ "targetType" : "LONG",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "YYYY_MM_DD",
+ "encoding" : "UTF-8"
+ } ]
+ }, {
+ "name" : "wholeTypeConverterConfigs",
+ "value" : [ {
+ "sourceType" : "INTEGER",
+ "targetType" : "INTEGER",
+ "treatInputFieldAsDate" : false,
+ "dataLocale" : "en,US",
+ "scale" : -1,
+ "decimalScaleRoundingStrategy" : "ROUND_UNNECESSARY",
+ "dateFormat" : "YYYY_MM_DD",
+ "encoding" : "UTF-8"
+ } ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Convert Datetime Fields",
+ "xPos" : 1600,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "FieldRemover_01OutputLane15030862999220" ],
+ "outputLanes" : [ "FieldTypeConverter_01OutputLane15030863815980" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HadoopFS_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hdfs_HdfsDTarget",
+ "stageVersion" : "4",
+ "configuration" : [ {
+ "name" : "hdfsTargetConfigBean.hdfsUri",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsUser",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsKerberos",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfDir",
+ "value" : "hadoop-conf"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.uniquePrefix",
+ "value" : "sdc-${sdc:id()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.fileNameSuffix",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplateInHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplate",
+ "value" : "${ODM_VULN_CTX_LOCATION}"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeDriver",
+ "value" : "${time:now()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.maxRecordsPerFile",
+ "value" : 0
+ }, {
+ "name" : "hdfsTargetConfigBean.maxFileSize",
+ "value" : 0
+ }, {
+ "name" : "hdfsTargetConfigBean.idleTimeout",
+ "value" : "${3 * SECONDS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "hdfsTargetConfigBean.otherCompression",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.fileType",
+ "value" : "TEXT"
+ }, {
+ "name" : "hdfsTargetConfigBean.keyEl",
+ "value" : "${uuid()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.seqFileCompressionType",
+ "value" : "BLOCK"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsLimit",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.rollIfHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.rollHeaderName",
+ "value" : "roll"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsAction",
+ "value" : "SEND_TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsDirPathTemplate",
+ "value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataFormat",
+ "value" : "AVRO"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsPermissionCheck",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.permissionEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.skipOldTempFileRecovery",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLines",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLinesString",
+ "value" : " "
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.jsonMode",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldPath",
+ "value" : "/text"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textRecordSeparator",
+ "value" : "\\n"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldMissingAction",
+ "value" : "ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textEmptyLineIfNull",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchemaSource",
+ "value" : "INLINE"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchema",
+ "value" : "{\n \"namespace\":\"org.apache.spot\",\n \"name\":\"vulnerability_context\",\n \"type\": \"record\",\n \"fields\": [\n {\"name\":\"vuln_id\",\"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_title\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_description\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_solution\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_type\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_category\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_severity\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_created\", \"type\":[\"null\",\"long\"],\"doc\":\"TBD\"},\n {\"name\":\"vuln_updated\", \"type\":[\"null\",\"long\"],\"doc\":\"TBD\"},\n {\"name\":\"additional_attrs\", \"type\":[\"null\",{
\"type\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null,\"doc\":\"Additional attributes of vulnerability\"}\n ],\n \"doc\": \"A view schema for storing Apache Spot Vulnerability Context data.\"\n }"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.registerSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subjectToRegister",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroCompression",
+ "value" : "NULL"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.binaryFieldPath",
+ "value" : "/"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.fileNameEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.wholeFileExistsAction",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.includeChecksumInTheEvents",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.checksumAlgorithm",
+ "value" : "MD5"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlPrettyPrint",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlValidateSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlSchema",
+ "value" : null
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Hadoop FS 1",
+ "xPos" : 1820,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ "FieldTypeConverter_01OutputLane15030863815980" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ "HadoopFS_01_EventLane" ]
+ }, {
+ "instanceName" : "MapReduce_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_mapreduce_MapReduceDExecutor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "mapReduceConfig.mapReduceConfDir",
+ "value" : "hadoop-conf"
+ }, {
+ "name" : "mapReduceConfig.mapreduceConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "mapReduceConfig.mapreduceUser",
+ "value" : null
+ }, {
+ "name" : "mapReduceConfig.kerberos",
+ "value" : false
+ }, {
+ "name" : "jobConfig.jobName",
+ "value" : "SDC MapReduceJob"
+ }, {
+ "name" : "jobConfig.jobType",
+ "value" : "AVRO_PARQUET"
+ }, {
+ "name" : "jobConfig.customJobCreator",
+ "value" : null
+ }, {
+ "name" : "jobConfig.jobConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "jobConfig.avroParquetConfig.inputFile",
+ "value" : "${record:value('/filepath')}"
+ }, {
+ "name" : "jobConfig.avroParquetConfig.outputDirectory",
+ "value" : "${ODM_VULN_CTX_PARQUET_LOCATION}"
+ }, {
+ "name" : "jobConfig.avroParquetConfig.keepInputFile",
+ "value" : false
+ }, {
+ "name" : "jobConfig.avroParquetConfig.compressionCodec",
+ "value" : null
+ }, {
+ "name" : "jobConfig.avroParquetConfig.rowGroupSize",
+ "value" : -1
+ }, {
+ "name" : "jobConfig.avroParquetConfig.pageSize",
+ "value" : -1
+ }, {
+ "name" : "jobConfig.avroParquetConfig.dictionaryPageSize",
+ "value" : -1
+ }, {
+ "name" : "jobConfig.avroParquetConfig.maxPaddingSize",
+ "value" : -1
+ }, {
+ "name" : "jobConfig.avroParquetConfig.overwriteTmpFile",
+ "value" : false
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "MapReduce 1",
+ "xPos" : 2040,
+ "yPos" : 200,
+ "stageType" : "EXECUTOR"
+ },
+ "inputLanes" : [ "HadoopFS_01_EventLane" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "errorStage" : {
+ "instanceName" : "Discard_ErrorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Error Records - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "info" : {
+ "pipelineId" : "QualysVulnerabilityContextTeste17011d3-a07e-43c8-a85a-0c403f9db901",
+ "title" : "ODMQualysVulnerabilityContext",
+ "description" : "",
+ "created" : 1505926790888,
+ "lastModified" : 1505938949506,
+ "creator" : "natty@dpmfield",
+ "lastModifier" : "natty@dpmfield",
+ "lastRev" : "0",
+ "uuid" : "97e5f9c0-3e2a-4686-b3d1-17eaf3e1d79b",
+ "valid" : true,
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "name" : "QualysVulnerabilityContextTeste17011d3-a07e-43c8-a85a-0c403f9db901",
+ "sdcVersion" : "2.7.1.0",
+ "sdcId" : "f0ff6a12-61b1-44a6-bc5f-0d7e67d7d482"
+ },
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "statsAggregatorStage" : {
+ "instanceName" : "WritetoDPMdirectly_StatsAggregatorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stats Aggregator - Write to DPM directly",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "startEventStages" : [ {
+ "instanceName" : "Discard_StartEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Start Event - Discard",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "stopEventStages" : [ {
+ "instanceName" : "Discard_StopEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stop Event - Discard",
+ "xPos" : 940,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "valid" : true,
+ "issues" : {
+ "issueCount" : 0,
+ "stageIssues" : { },
+ "pipelineIssues" : [ ]
+ },
+ "previewable" : true
+ },
+ "pipelineRules" : {
+ "schemaVersion" : 3,
+ "version" : 2,
+ "metricsRuleDefinitions" : [ {
+ "id" : "badRecordsAlertID",
+ "alertText" : "High incidence of Error Records",
+ "metricId" : "pipeline.batchErrorRecords.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "stageErrorAlertID",
+ "alertText" : "High incidence of Stage Errors",
+ "metricId" : "pipeline.batchErrorMessages.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "idleGaugeID",
+ "alertText" : "Pipeline is Idle",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
+ "condition" : "${time:now() - value() > 120000}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "batchTimeAlertID",
+ "alertText" : "Batch taking more time to process",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "CURRENT_BATCH_AGE",
+ "condition" : "${value() > 200}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ }, {
+ "id" : "memoryLimitAlertID",
+ "alertText" : "Memory limit for pipeline exceeded",
+ "metricId" : "pipeline.memoryConsumed.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > (jvm:maxMemoryMB() * 0.65)}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1502402628465,
+ "valid" : true
+ } ],
+ "dataRuleDefinitions" : [ ],
+ "driftRuleDefinitions" : [ ],
+ "uuid" : "3e7661fe-4244-4c78-995d-8aa99ccf6cff",
+ "configuration" : [ {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ } ],
+ "ruleIssues" : [ ],
+ "configIssues" : [ ]
+ },
+ "libraryDefinitions" : null
+}
\ No newline at end of file
[6/7] incubator-spot git commit: Add README for StreamSets pipelines
Posted by na...@apache.org.
Add README for StreamSets pipelines
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/08d6b522
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/08d6b522
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/08d6b522
Branch: refs/heads/SPOT-181_ODM
Commit: 08d6b522bebe58a3a7a6f3086155d20bec8d1287
Parents: b6bba64
Author: Jon Natkins <na...@streamsets.com>
Authored: Fri Sep 22 11:48:12 2017 -0700
Committer: Jon Natkins <na...@streamsets.com>
Committed: Fri Sep 22 11:48:12 2017 -0700
----------------------------------------------------------------------
spot-ingest/streamsets/README.md | 27 +++++++++++++++++++
spot-ingest/streamsets/images/ImportContext.png | Bin 0 -> 61789 bytes
.../streamsets/images/ImportPipeline.png | Bin 0 -> 65915 bytes
3 files changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/08d6b522/spot-ingest/streamsets/README.md
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/README.md b/spot-ingest/streamsets/README.md
new file mode 100644
index 0000000..9191807
--- /dev/null
+++ b/spot-ingest/streamsets/README.md
@@ -0,0 +1,27 @@
+# StreamSets Ingest Pipelines for Apache Spot ODM
+
+The [StreamSets Data Collector](https://github.com/streamsets/datacollector) is an Apache 2.0-licensed open-source tool for data ingestion pipeline development and execution.
+
+### What's Here?
+The subdirectories contain JSON pipeline configurations that can be imported into the StreamSets Data Collector, and configured to continuously ingest data into Avro-stored ODM Hive tables.
+
+As of 9/22/2017, the following data sources are supported:
+ - Qualys Knowledge Base
+ - Qualys Vulernability Scans
+ - Windows Security Logs
+ - Centrify Identity Platform Logs
+
+### Requirements
+ - SDC 2.7.1.1 or newer: Pipelines have been built on SDC 2.7.1.1, and will work with any newer SDC version. For older SDC versions, pipelines may not import smoothly and may require re-development. A full install of SDC is recommended, as the core tarball installation does not include all required stage libraries.
+ - Oracle Java 8
+
+### Importing Pipelines
+StreamSets pipeline configurations can be imported into an SDC via the REST API, CLI, or through the UI. After downloading the pipeline configurations, import the JSON files:
+
+<img src="images/ImportContext.png" width="500">
+
+<img src="images/ImportPipeline.png" width="500">
+
+### License
+Apache Software License 2.0
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/08d6b522/spot-ingest/streamsets/images/ImportContext.png
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/images/ImportContext.png b/spot-ingest/streamsets/images/ImportContext.png
new file mode 100644
index 0000000..998aa29
Binary files /dev/null and b/spot-ingest/streamsets/images/ImportContext.png differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/08d6b522/spot-ingest/streamsets/images/ImportPipeline.png
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/images/ImportPipeline.png b/spot-ingest/streamsets/images/ImportPipeline.png
new file mode 100644
index 0000000..bae8ef4
Binary files /dev/null and b/spot-ingest/streamsets/images/ImportPipeline.png differ
[7/7] incubator-spot git commit: SPOT-226 closes
apache/incubator-spot#118
Posted by na...@apache.org.
SPOT-226 closes apache/incubator-spot#118
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/d9232593
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/d9232593
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/d9232593
Branch: refs/heads/SPOT-181_ODM
Commit: d9232593ed1dbadcca484f523523124ea3ab62a6
Parents: 392e2a9 08d6b52
Author: natedogs911 <na...@gmail.com>
Authored: Fri Sep 29 09:53:11 2017 -0700
Committer: natedogs911 <na...@gmail.com>
Committed: Fri Sep 29 09:53:11 2017 -0700
----------------------------------------------------------------------
spot-ingest/streamsets/README.md | 27 +
.../ODMCentrifyIdentityPlatformEventTCP.json | 1096 +++++++++++++++
spot-ingest/streamsets/images/ImportContext.png | Bin 0 -> 61789 bytes
.../streamsets/images/ImportPipeline.png | Bin 0 -> 65915 bytes
.../qualys/ODMQualysVulnerabilityContext.json | 1276 ++++++++++++++++++
.../qualys/ODMQualysVulnerabilityEvents.json | 1245 +++++++++++++++++
.../streamsets/windows/ODMWindowsEventLogs.json | 914 +++++++++++++
7 files changed, 4558 insertions(+)
----------------------------------------------------------------------
[3/7] incubator-spot git commit: SPOT-228. [Ingest] StreamSets
pipeline configs for Windows event logs
Posted by na...@apache.org.
SPOT-228. [Ingest] StreamSets pipeline configs for Windows event logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/ed388f75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/ed388f75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/ed388f75
Branch: refs/heads/SPOT-181_ODM
Commit: ed388f7544df9997919e3063d872f46149f07205
Parents: fe5862c
Author: Jon Natkins <na...@streamsets.com>
Authored: Wed Sep 20 22:46:44 2017 -0700
Committer: Jon Natkins <na...@streamsets.com>
Committed: Thu Sep 21 13:43:23 2017 -0700
----------------------------------------------------------------------
.../streamsets/windows/ODMWindowsEventLogs.json | 914 +++++++++++++++++++
1 file changed, 914 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/ed388f75/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
----------------------------------------------------------------------
diff --git a/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json b/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
new file mode 100644
index 0000000..e5a8eaa
--- /dev/null
+++ b/spot-ingest/streamsets/windows/ODMWindowsEventLogs.json
@@ -0,0 +1,914 @@
+{
+ "pipelineConfig" : {
+ "schemaVersion" : 4,
+ "version" : 7,
+ "pipelineId" : "ODMWindowsEventLogs9fae6da4-eb7f-4aca-820a-1f561a40e7cc",
+ "title" : "ODMWindowsEventLogs",
+ "description" : "",
+ "uuid" : "a155ab31-89d9-44d9-94d3-c06fc1fda992",
+ "configuration" : [ {
+ "name" : "executionMode",
+ "value" : "STANDALONE"
+ }, {
+ "name" : "deliveryGuarantee",
+ "value" : "AT_LEAST_ONCE"
+ }, {
+ "name" : "startEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "stopEventStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "shouldRetry",
+ "value" : true
+ }, {
+ "name" : "retryAttempts",
+ "value" : -1
+ }, {
+ "name" : "memoryLimit",
+ "value" : "${jvm:maxMemoryMB() * 0.65}"
+ }, {
+ "name" : "memoryLimitExceeded",
+ "value" : "STOP_PIPELINE"
+ }, {
+ "name" : "notifyOnStates",
+ "value" : [ "RUN_ERROR", "STOPPED", "FINISHED" ]
+ }, {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "constants",
+ "value" : [ {
+ "key" : "ODM_EVENTS_LOCATION",
+ "value" : ""
+ }, {
+ "key" : "ODM_EVENTS_TABLE_NAME",
+ "value" : ""
+ }, {
+ "key" : "HIVE_URL",
+ "value" : ""
+ }, {
+ "key" : "WINDOWS_EVENT_DIR",
+ "value" : ""
+ } ]
+ }, {
+ "name" : "badRecordsHandling",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget::1"
+ }, {
+ "name" : "errorRecordPolicy",
+ "value" : "ORIGINAL_RECORD"
+ }, {
+ "name" : "workerCount",
+ "value" : 0
+ }, {
+ "name" : "clusterSlaveMemory",
+ "value" : 1024
+ }, {
+ "name" : "clusterSlaveJavaOpts",
+ "value" : "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -Dlog4j.debug"
+ }, {
+ "name" : "clusterLauncherEnv",
+ "value" : [ ]
+ }, {
+ "name" : "mesosDispatcherURL",
+ "value" : null
+ }, {
+ "name" : "hdfsS3ConfDir",
+ "value" : null
+ }, {
+ "name" : "rateLimit",
+ "value" : 0
+ }, {
+ "name" : "maxRunners",
+ "value" : 0
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "sparkConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "statsAggregatorStage",
+ "value" : "streamsets-datacollector-basic-lib::com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget::1"
+ } ],
+ "uiInfo" : {
+ "previewConfig" : {
+ "previewSource" : "CONFIGURED_SOURCE",
+ "batchSize" : 10,
+ "timeout" : 10000,
+ "writeToDestinations" : false,
+ "executeLifecycleEvents" : false,
+ "showHeader" : false,
+ "showFieldType" : true,
+ "rememberMe" : false
+ }
+ },
+ "stages" : [ {
+ "instanceName" : "Directory_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_origin_spooldir_SpoolDirDSource",
+ "stageVersion" : "9",
+ "configuration" : [ {
+ "name" : "conf.dataFormatConfig.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.dataFormatConfig.filePatternInArchive",
+ "value" : "*"
+ }, {
+ "name" : "conf.dataFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "conf.dataFormatConfig.removeCtrlChars",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.textMaxLineLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.useCustomDelimiter",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customDelimiter",
+ "value" : "\\r\\n"
+ }, {
+ "name" : "conf.dataFormatConfig.includeCustomDelimiterInTheText",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.jsonContent",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "conf.dataFormatConfig.jsonMaxObjectLen",
+ "value" : 4096000
+ }, {
+ "name" : "conf.dataFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "conf.dataFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "conf.dataFormatConfig.csvMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.csvAllowExtraColumns",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvExtraColumnPrefix",
+ "value" : "_extra_"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "conf.dataFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "conf.dataFormatConfig.csvEnableComments",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.csvCommentMarker",
+ "value" : "#"
+ }, {
+ "name" : "conf.dataFormatConfig.csvIgnoreEmptyLines",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.csvRecordType",
+ "value" : "LIST_MAP"
+ }, {
+ "name" : "conf.dataFormatConfig.csvSkipStartLines",
+ "value" : 0
+ }, {
+ "name" : "conf.dataFormatConfig.parseNull",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.nullConstant",
+ "value" : "\\\\N"
+ }, {
+ "name" : "conf.dataFormatConfig.xmlRecordElement",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.includeFieldXpathAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xPathNamespaceContext",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.outputFieldAttributes",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.xmlMaxObjectLen",
+ "value" : 4096
+ }, {
+ "name" : "conf.dataFormatConfig.logMode",
+ "value" : "COMMON_LOG_FORMAT"
+ }, {
+ "name" : "conf.dataFormatConfig.logMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.retainOriginalLine",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.customLogFormat",
+ "value" : "%h %l %u %t \"%r\" %>s %b"
+ }, {
+ "name" : "conf.dataFormatConfig.regex",
+ "value" : "^(\\S+) (\\S+) (\\S+) \\[([\\w:/]+\\s[+\\-]\\d{4})\\] \"(\\S+) (\\S+) (\\S+)\" (\\d{3}) (\\d+)"
+ }, {
+ "name" : "conf.dataFormatConfig.fieldPathsToGroupName",
+ "value" : [ {
+ "fieldPath" : "/",
+ "group" : 1
+ } ]
+ }, {
+ "name" : "conf.dataFormatConfig.grokPatternDefinition",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.grokPattern",
+ "value" : "%{COMMONAPACHELOG}"
+ }, {
+ "name" : "conf.dataFormatConfig.onParseError",
+ "value" : "ERROR"
+ }, {
+ "name" : "conf.dataFormatConfig.maxStackTraceLines",
+ "value" : 50
+ }, {
+ "name" : "conf.dataFormatConfig.enableLog4jCustomLogFormat",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.log4jCustomLogFormat",
+ "value" : "%r [%t] %-5p %c %x - %m%n"
+ }, {
+ "name" : "conf.dataFormatConfig.avroSchema",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "conf.dataFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "conf.dataFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.isDelimited",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.binaryMaxObjectLen",
+ "value" : 1024
+ }, {
+ "name" : "conf.dataFormatConfig.datagramMode",
+ "value" : "SYSLOG"
+ }, {
+ "name" : "conf.dataFormatConfig.typesDbPath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.convertTime",
+ "value" : false
+ }, {
+ "name" : "conf.dataFormatConfig.excludeInterval",
+ "value" : true
+ }, {
+ "name" : "conf.dataFormatConfig.authFilePath",
+ "value" : null
+ }, {
+ "name" : "conf.dataFormatConfig.wholeFileMaxObjectLen",
+ "value" : 8192
+ }, {
+ "name" : "conf.dataFormatConfig.rateLimit",
+ "value" : "-1"
+ }, {
+ "name" : "conf.dataFormat",
+ "value" : "JSON"
+ }, {
+ "name" : "conf.spoolDir",
+ "value" : "${WINDOWS_EVENT_DIR}"
+ }, {
+ "name" : "conf.pathMatcherMode",
+ "value" : "GLOB"
+ }, {
+ "name" : "conf.filePattern",
+ "value" : "*"
+ }, {
+ "name" : "conf.useLastModified",
+ "value" : "TIMESTAMP"
+ }, {
+ "name" : "conf.processSubdirectories",
+ "value" : false
+ }, {
+ "name" : "conf.allowLateDirectory",
+ "value" : false
+ }, {
+ "name" : "conf.overrunLimit",
+ "value" : 128
+ }, {
+ "name" : "conf.batchSize",
+ "value" : 1000
+ }, {
+ "name" : "conf.poolingTimeoutSecs",
+ "value" : 600
+ }, {
+ "name" : "conf.maxSpoolFiles",
+ "value" : 1000
+ }, {
+ "name" : "conf.initialFileToProcess",
+ "value" : null
+ }, {
+ "name" : "conf.errorArchiveDir",
+ "value" : null
+ }, {
+ "name" : "conf.postProcessing",
+ "value" : "NONE"
+ }, {
+ "name" : "conf.archiveDir",
+ "value" : null
+ }, {
+ "name" : "conf.retentionTimeMins",
+ "value" : 0
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ } ],
+ "uiInfo" : {
+ "yPos" : 50,
+ "stageType" : "SOURCE",
+ "rawSource" : {
+ "configuration" : [ {
+ "name" : "fileName"
+ } ]
+ },
+ "description" : "",
+ "label" : "Read Windows Event Files",
+ "xPos" : 60
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ "Directory_01OutputLane15059716464100" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ {
+ "fieldToSet" : "/dvc_host",
+ "expression" : "${record:value('/ComputerName')}"
+ }, {
+ "fieldToSet" : "/event_time",
+ "expression" : "${record:value('/TimeGenerated') * 1000}"
+ }, {
+ "fieldToSet" : "/msg",
+ "expression" : "${record:value('/Message')}"
+ }, {
+ "fieldToSet" : "/name",
+ "expression" : "${record:value('/LogName')}"
+ }, {
+ "fieldToSet" : "/category",
+ "expression" : "${record:value('/Category')}"
+ }, {
+ "fieldToSet" : "/event_id",
+ "expression" : "${record:value('/EventId')}"
+ }, {
+ "fieldToSet" : "/user",
+ "expression" : "${record:value('/User')}"
+ } ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ {
+ "attributeToSet" : "p_dvc_vendor",
+ "headerAttributeExpression" : "Microsoft"
+ }, {
+ "attributeToSet" : "p_dvc_type",
+ "headerAttributeExpression" : "${record:value('/SourceName')}"
+ }, {
+ "attributeToSet" : "p_dt",
+ "headerAttributeExpression" : "${time:extractStringFromDate(time:now(),'YYYY-MM-dd')}"
+ } ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Map to ODM",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "Directory_01OutputLane15059716464100" ],
+ "outputLanes" : [ "ExpressionEvaluator_01OutputLane15059452532370" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "FieldRemover_01",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_fieldfilter_FieldFilterDProcessor",
+ "stageVersion" : "1",
+ "configuration" : [ {
+ "name" : "filterOperation",
+ "value" : "KEEP"
+ }, {
+ "name" : "fields",
+ "value" : [ "/msg", "/name", "/category", "/dvc_host", "/event_id", "/user", "/event_time" ]
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Field Remover 1",
+ "xPos" : 500,
+ "yPos" : 50,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_01OutputLane15059452532370" ],
+ "outputLanes" : [ "FieldRemover_01OutputLane15059461007910" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HadoopFS_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hdfs_HdfsDTarget",
+ "stageVersion" : "4",
+ "configuration" : [ {
+ "name" : "hdfsTargetConfigBean.hdfsUri",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsUser",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsKerberos",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfDir",
+ "value" : "hadoop-conf"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.uniquePrefix",
+ "value" : "sdc-${sdc:id()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.fileNameSuffix",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplateInHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dirPathTemplate",
+ "value" : "${ODM_EVENTS_LOCATION}/p_dvc_type='${record:attribute(\"p_dvc_type\")}'/p_dvc_vendor='${record:attribute(\"p_dvc_vendor\")}'/p_dt='${record:attribute(\"p_dt\")}'"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeZoneID",
+ "value" : "UTC"
+ }, {
+ "name" : "hdfsTargetConfigBean.timeDriver",
+ "value" : "${time:now()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.maxRecordsPerFile",
+ "value" : 1
+ }, {
+ "name" : "hdfsTargetConfigBean.maxFileSize",
+ "value" : 0
+ }, {
+ "name" : "hdfsTargetConfigBean.idleTimeout",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.compression",
+ "value" : "NONE"
+ }, {
+ "name" : "hdfsTargetConfigBean.otherCompression",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.fileType",
+ "value" : "TEXT"
+ }, {
+ "name" : "hdfsTargetConfigBean.keyEl",
+ "value" : "${uuid()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.seqFileCompressionType",
+ "value" : "BLOCK"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsLimit",
+ "value" : "${1 * HOURS}"
+ }, {
+ "name" : "hdfsTargetConfigBean.rollIfHeader",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.rollHeaderName",
+ "value" : "roll"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsAction",
+ "value" : "SEND_TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.lateRecordsDirPathTemplate",
+ "value" : "/tmp/late/${YYYY()}-${MM()}-${DD()}"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataFormat",
+ "value" : "AVRO"
+ }, {
+ "name" : "hdfsTargetConfigBean.hdfsPermissionCheck",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.permissionEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.skipOldTempFileRecovery",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.charset",
+ "value" : "UTF-8"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvFileFormat",
+ "value" : "CSV"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvHeader",
+ "value" : "NO_HEADER"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLines",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvReplaceNewLinesString",
+ "value" : " "
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomDelimiter",
+ "value" : "|"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomEscape",
+ "value" : "\\"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.csvCustomQuote",
+ "value" : "\""
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.jsonMode",
+ "value" : "MULTIPLE_OBJECTS"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldPath",
+ "value" : "/text"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textRecordSeparator",
+ "value" : "\\n"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textFieldMissingAction",
+ "value" : "ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.textEmptyLineIfNull",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchemaSource",
+ "value" : "INLINE"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroSchema",
+ "value" : "{\n\n \"namespace\":\"org.apache.spot\",\n \"name\":\"event\",\n \"type\": \"record\",\n \"fields\": [\n {\"name\":\"event_time\",\"type\":[\"null\",\"long\"],\"doc\":\"timestamp of event (UTC)\", \"default\": null},\n {\"name\":\"duration\", \"type\":[\"null\",\"float\"],\"doc\":\"Time duration (milliseconds)\", \"default\": null},\n {\"name\":\"event_id\", \"type\":[\"null\",\"string\"],\"doc\":\"Unique identifier for event\", \"default\": null},\n {\"name\":\"name\", \"type\":[\"null\",\"string\"],\"doc\":\"Name of event\", \"default\": null},\n {\"name\":\"org\", \"type\":[\"null\",\"string\"],\"doc\":\"Organization\", \"default\": null},\n {\"name\":\"type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type information\", \"default\": null},\n {\"name\":\"n_proto\", \"type\":[\"null\",\"string\"],\"doc\":\"Network protocol of event\", \"default\": null},\n {\"name\":\"a_proto\"
, \"type\":[\"null\",\"string\"],\"doc\":\"Application protocol of event\", \"default\": null},\n {\"name\":\"msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Message (details of action taken on object)\", \"default\": null},\n {\"name\":\"mac\", \"type\":[\"null\",\"string\"],\"doc\":\"MAC address\", \"default\": null},\n {\"name\":\"severity\", \"type\":[\"null\",\"string\"],\"doc\":\"Severity of event\", \"default\": null},\n {\"name\":\"raw\", \"type\":[\"null\",\"string\"],\"doc\":\"Raw text message of entire event\", \"default\": null},\n {\"name\":\"risk\", \"type\":[\"null\",\"float\"],\"doc\":\"Risk score\", \"default\": null},\n {\"name\":\"code\", \"type\":[\"null\",\"string\"],\"doc\":\"Response or error code\", \"default\": null},\n {\"name\":\"category\", \"type\":[\"null\",\"string\"],\"doc\":\"Event category\", \"default\": null},\n {\"name\":\"query\", \"type\":[\"null\",\"string\"],\"doc\":\"Que
ry (DNS query, URI query, SQL query, etc.)\", \"default\": null},\n {\"name\":\"service\", \"type\":[\"null\",\"string\"],\"doc\":\"(i.e. service name, type of service)\", \"default\": null},\n {\"name\":\"state\", \"type\":[\"null\",\"string\"],\"doc\":\"State of object\", \"default\": null},\n {\"name\":\"in_bytes\", \"type\":[\"null\",\"int\"],\"doc\":\"Bytes in\", \"default\": null},\n {\"name\":\"out_bytes\", \"type\":[\"null\",\"int\"],\"doc\":\"Bytes out\", \"default\": null},\n {\"name\":\"xref\", \"type\":[\"null\",\"string\"],\"doc\":\"External reference to public description\", \"default\": null},\n {\"name\":\"version\", \"type\":[\"null\",\"string\"],\"doc\":\"Version\", \"default\": null},\n {\"name\":\"additional_attrs\",\"type\":[\"null\",{\"type\":\"map\",\"values\":[\"null\",\"string\"]}],\"default\":null, \"doc\":\"Additional attributes of the event\", \"default\": null},\n {\"name\":\"dvc_time\
", \"type\":[\"null\",\"long\"],\"doc\":\"UTC timestamp from device where event/alert originates or is received\", \"default\": null},\n {\"name\":\"dvc_ip4\", \"type\":[\"null\",\"string\"],\"doc\":\"IP address of device\", \"default\": null},\n {\"name\":\"dvc_ip6\", \"type\":[\"null\",\"string\"],\"doc\":\"IP address of device\", \"default\": null},\n {\"name\":\"dvc_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Hostname of device\", \"default\": null},\n {\"name\":\"dvc_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Device type that generated the log\", \"default\": null},\n {\"name\":\"dvc_vendor\", \"type\":[\"null\",\"string\"],\"doc\":\"Vendor\", \"default\": null},\n {\"name\":\"dvc_fwd_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Forwarded from device\", \"default\": null},\n {\"name\":\"dvc_fwd_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Forwarded from device\", \"default\": null},\n {\"name\":\"dvc_version\"
, \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"src_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Source ip address of event\", \"default\": null},\n {\"name\":\"src_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Source ip address of event\", \"default\": null},\n {\"name\":\"src_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Source FQDN of event\", \"default\": null},\n {\"name\":\"src_domain\", \"type\":[\"null\",\"string\"],\"doc\":\"Domain name of source address\", \"default\": null},\n {\"name\":\"src_port\", \"type\":[\"null\",\"int\"],\"doc\":\"Source port of event\", \"default\": null},\n {\"name\":\"src_country_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country code\", \"default\": null},\n {\"name\":\"src_country_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country name\", \"default\": null},\n {\"name\":\"src_region\", \"type\":[\"null\",\"string\"],\"doc\"
:\"Source region\", \"default\": null},\n {\"name\":\"src_city\", \"type\":[\"null\",\"string\"],\"doc\":\"Source city\", \"default\": null},\n {\"name\":\"src_lat\", \"type\":[\"null\",\"int\"],\"doc\":\"Source latitude\", \"default\": null},\n {\"name\":\"src_long\", \"type\":[\"null\",\"int\"],\"doc\":\"Source longitude\", \"default\": null},\n {\"name\":\"dst_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Destination ip address of event\", \"default\": null},\n {\"name\":\"dst_ip6\", \"type\":[\"null\",\"long\"],\"doc\":\"Destination ip address of event\", \"default\": null},\n {\"name\":\"dst_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Destination FQDN of event\", \"default\": null},\n {\"name\":\"dst_domain\", \"type\":[\"null\",\"string\"],\"doc\":\"Domain name of destination address\", \"default\": null},\n {\"name\":\"dst_port\", \"type\":[\"null\",\"int\"],\"doc\":\"Destination port of event\", \"default\": null},
\n {\"name\":\"dst_country_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country code\", \"default\": null},\n {\"name\":\"dst_country_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Source country name\", \"default\": null},\n {\"name\":\"dst_region\", \"type\":[\"null\",\"string\"],\"doc\":\"Source region\", \"default\": null},\n {\"name\":\"dst_city\", \"type\":[\"null\",\"string\"],\"doc\":\"Source city\", \"default\": null},\n {\"name\":\"dst_lat\", \"type\":[\"null\",\"int\"],\"doc\":\"Source latitude\", \"default\": null},\n {\"name\":\"dst_long\", \"type\":[\"null\",\"int\"],\"doc\":\"Source longitude\", \"default\": null},\n {\"name\":\"src_asn\", \"type\":[\"null\",\"int\"],\"doc\":\"Autonomous system number\", \"default\": null},\n {\"name\":\"dst_asn\", \"type\":[\"null\",\"int\"],\"doc\":\"Autonomous system number\", \"default\": null},\n {\"name\":\"net_direction\", \"type\":[\"null\",\"string\"]
,\"doc\":\"Direction\", \"default\": null},\n {\"name\":\"net_flags\", \"type\":[\"null\",\"string\"],\"doc\":\"TCP flags\", \"default\": null},\n {\"name\":\"file_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Filename from event\", \"default\": null},\n {\"name\":\"file_path\", \"type\":[\"null\",\"string\"],\"doc\":\"File path\", \"default\": null},\n {\"name\":\"file_atime\", \"type\":[\"null\",\"long\"],\"doc\":\"Timestamp (UTC) of file access\", \"default\": null},\n {\"name\":\"file_acls\", \"type\":[\"null\",\"string\"],\"doc\":\"File permissions\", \"default\": null},\n {\"name\":\"file_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of file\", \"default\": null},\n {\"name\":\"file_size\", \"type\":[\"null\",\"int\"],\"doc\":\"Size of file in bytes\", \"default\": null},\n {\"name\":\"file_desc\", \"type\":[\"null\",\"string\"],\"doc\":\"Description of file\", \"default\": null},\n {\"name\":\"file_hash
\", \"type\":[\"null\",\"string\"],\"doc\":\"Hash of file\", \"default\": null},\n {\"name\":\"file_hash_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of hash\", \"default\": null},\n {\"name\":\"end_object\", \"type\":[\"null\",\"string\"],\"doc\":\"File/Process/ Registry\", \"default\": null},\n {\"name\":\"end_action\", \"type\":[\"null\",\"string\"],\"doc\":\"Action taken on object (open/delete/ edit)\", \"default\": null},\n {\"name\":\"end_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Message (details of action taken on object)\", \"default\": null},\n {\"name\":\"end_app\", \"type\":[\"null\",\"string\"],\"doc\":\"Application\", \"default\": null},\n {\"name\":\"end_location\", \"type\":[\"null\",\"string\"],\"doc\":\"Location\", \"default\": null},\n {\"name\":\"end_proc\", \"type\":[\"null\",\"string\"],\"doc\":\"Process\", \"default\": null},\n {\"name\":\"user_name\", \"type\":[\"null\",\"string\"],\"doc\":
\"username from event\", \"default\": null},\n {\"name\":\"src_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"dst_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"user_email\", \"type\":[\"null\",\"string\"],\"doc\":\"Email address\", \"default\": null},\n {\"name\":\"user_id\", \"type\":[\"null\",\"string\"],\"doc\":\"userid\", \"default\": null},\n {\"name\":\"user_loc\", \"type\":[\"null\",\"string\"],\"doc\":\"location\", \"default\": null},\n {\"name\":\"user_desc\", \"type\":[\"null\",\"string\"],\"doc\":\"Description of user\", \"default\": null},\n {\"name\":\"dns_class\", \"type\":[\"null\",\"string\"],\"doc\":\"DNS class\", \"default\": null},\n {\"name\":\"dns_len\", \"type\":[\"null\",\"int\"],\"doc\":\"DNS frame length\", \"default\": null},\n {\"name\":\"dns_query\", \"type\":[\"null
\",\"string\"],\"doc\":\"Requested DNS query\", \"default\": null},\n {\"name\":\"dns_response_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Response code\", \"default\": null},\n {\"name\":\"dns_answers\", \"type\":[\"null\",\"string\"],\"doc\":\"Response to DNS Query\", \"default\": null},\n {\"name\":\"dns_type\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"prx_category\", \"type\":[\"null\",\"string\"],\"doc\":\"Event category\", \"default\": null},\n {\"name\":\"prx_browser\", \"type\":[\"null\",\"string\"],\"doc\":\"Web browser\", \"default\": null},\n {\"name\":\"prx_code\", \"type\":[\"null\",\"string\"],\"doc\":\"Error or response code\", \"default\": null},\n {\"name\":\"prx_referrer\", \"type\":[\"null\",\"string\"],\"doc\":\"Referrer\", \"default\": null},\n {\"name\":\"prx_host\", \"type\":[\"null\",\"string\"],\"doc\":\"Requested URI\", \"default\": null},\n {\"name\":\
"prx_filter_rule\", \"type\":[\"null\",\"string\"],\"doc\":\"Applied filter or rule\", \"default\": null},\n {\"name\":\"prx_filter_result\", \"type\":[\"null\",\"string\"],\"doc\":\"Result of applied filter or rule\", \"default\": null},\n {\"name\":\"prx_query\", \"type\":[\"null\",\"string\"],\"doc\":\"URI query\", \"default\": null},\n {\"name\":\"prx_action\", \"type\":[\"null\",\"string\"],\"doc\":\"Action taken on object\", \"default\": null},\n {\"name\":\"prx_method\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP method\", \"default\": null},\n {\"name\":\"prx_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Type of request\", \"default\": null},\n {\"name\":\"http_request_method\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP method\", \"default\": null},\n {\"name\":\"http_request_uri\", \"type\":[\"null\",\"string\"],\"doc\":\"Requested URI\", \"default\": null},\n {\"name\":\"http_request_body_len\", \"type\":[\"
null\",\"int\"],\"doc\":\"Length of request body\", \"default\": null},\n {\"name\":\"http_request_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"username from event\", \"default\": null},\n {\"name\":\"http_request_password\", \"type\":[\"null\",\"string\"],\"doc\":\"Password from event\", \"default\": null},\n {\"name\":\"http_request_proxied\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"http_request_headers\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP request headers\", \"default\": null},\n {\"name\":\"http_response_status_code\", \"type\":[\"null\",\"int\"],\"doc\":\"HTTP response status code\", \"default\": null},\n {\"name\":\"http_response_status_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP response status message\", \"default\": null},\n {\"name\":\"http_response_body_len\", \"type\":[\"null\",\"string\"],\"doc\":\"Length of response body\", \"default\": null},\n
{\"name\":\"http_response_info_code\", \"type\":[\"null\",\"int\"],\"doc\":\"HTTP response info code\", \"default\": null},\n {\"name\":\"http_response_info_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"HTTP response info message\", \"default\": null},\n {\"name\":\"http_response_resp_fuids\", \"type\":[\"null\",\"string\"],\"doc\":\"Response FUIDS\", \"default\": null},\n {\"name\":\"http_response_mime_types\", \"type\":[\"null\",\"string\"],\"doc\":\"Mime types\", \"default\": null},\n {\"name\":\"http_response_headers\", \"type\":[\"null\",\"string\"],\"doc\":\"Response headers\", \"default\": null},\n {\"name\":\"smtp_trans_depth\", \"type\":[\"null\",\"int\"],\"doc\":\"Depth of email into SMTP exchange\", \"default\": null},\n {\"name\":\"smtp_headers_helo\", \"type\":[\"null\",\"string\"],\"doc\":\"Helo header\", \"default\": null},\n {\"name\":\"smtp_headers_mailfrom\", \"type\":[\"null\",\"string\"],\"doc\":\"Mailfrom head
er\", \"default\": null},\n {\"name\":\"smtp_headers_rcptto\", \"type\":[\"null\",\"string\"],\"doc\":\"Rcptto header\", \"default\": null},\n {\"name\":\"smtp_headers_date\", \"type\":[\"null\",\"string\"],\"doc\":\"Header date\", \"default\": null},\n {\"name\":\"smtp_headers_from\", \"type\":[\"null\",\"string\"],\"doc\":\"From header\", \"default\": null},\n {\"name\":\"smtp_headers_to\", \"type\":[\"null\",\"string\"],\"doc\":\"To header\", \"default\": null},\n {\"name\":\"smtp_headers_reply_to\", \"type\":[\"null\",\"string\"],\"doc\":\"Reply to header\", \"default\": null},\n {\"name\":\"smtp_headers_msg_id\", \"type\":[\"null\",\"string\"],\"doc\":\"Message ID\", \"default\": null},\n {\"name\":\"smtp_headers_in_reply_to\", \"type\":[\"null\",\"string\"],\"doc\":\"In reply to header\", \"default\": null},\n {\"name\":\"smpt_headers_subject\", \"type\":[\"null\",\"string\"],\"doc\":\"Subject\", \"default\": null},\n
{\"name\":\"smtp_headers_x_originating_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"Originating IP address\", \"default\": null},\n {\"name\":\"smtp_headers_first_received\", \"type\":[\"null\",\"string\"],\"doc\":\"First to receive message\", \"default\": null},\n {\"name\":\"smtp_headers_second_received\", \"type\":[\"null\",\"string\"],\"doc\":\"Second to receive message\", \"default\": null},\n {\"name\":\"smtp_last_reply\", \"type\":[\"null\",\"string\"],\"doc\":\"Last reply in message chain\", \"default\": null},\n {\"name\":\"smtp_path\", \"type\":[\"null\",\"string\"],\"doc\":\"Path of message\", \"default\": null},\n {\"name\":\"smtp_user_agent\", \"type\":[\"null\",\"string\"],\"doc\":\"User agent\", \"default\": null},\n {\"name\":\"smtp_tls\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Indication of TLS use\", \"default\": null},\n {\"name\":\"smtp_is_webmail\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Indication of w
ebmail\", \"default\": null},\n {\"name\":\"ftp_user_name\", \"type\":[\"null\",\"string\"],\"doc\":\"Username\", \"default\": null},\n {\"name\":\"ftp_password\", \"type\":[\"null\",\"string\"],\"doc\":\"Password\", \"default\": null},\n {\"name\":\"ftp_command\", \"type\":[\"null\",\"string\"],\"doc\":\"FTP command\", \"default\": null},\n {\"name\":\"ftp_arg\", \"type\":[\"null\",\"string\"],\"doc\":\"Argument\", \"default\": null},\n {\"name\":\"ftp_mime_type\", \"type\":[\"null\",\"string\"],\"doc\":\"Mime type\", \"default\": null},\n {\"name\":\"ftp_file_size\", \"type\":[\"null\",\"int\"],\"doc\":\"File size\", \"default\": null},\n {\"name\":\"ftp_reply_code\", \"type\":[\"null\",\"int\"],\"doc\":\"Reply code\", \"default\": null},\n {\"name\":\"ftp_reply_msg\", \"type\":[\"null\",\"string\"],\"doc\":\"Reply message\", \"default\": null},\n {\"name\":\"ftp_data_channel_passive\", \"type\":[\"null\",\"boolean\"],
\"doc\":\"Passive data channel?\", \"default\": null},\n {\"name\":\"ftp_data_channel_rsp_p\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_cwd\", \"type\":[\"null\",\"string\"],\"doc\":\"Current working directory\", \"default\": null},\n {\"name\":\"ftp_cmdarg_ts\", \"type\":[\"null\",\"float\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_cmdarg_cmd\", \"type\":[\"null\",\"string\"],\"doc\":\"Command\", \"default\": null},\n {\"name\":\"ftp_cmdarg_arg\", \"type\":[\"null\",\"string\"],\"doc\":\"Command argument\", \"default\": null},\n {\"name\":\"ftp_cmdarg_seq\", \"type\":[\"null\",\"int\"],\"doc\":\"Sequence\", \"default\": null},\n {\"name\":\"ftp_pending_commands\", \"type\":[\"null\",\"string\"],\"doc\":\"Pending commands\", \"default\": null},\n {\"name\":\"ftp_is_passive\", \"type\":[\"null\",\"boolean\"],\"doc\":\"Passive mode enabled\", \"default\": null},\n
{\"name\":\"ftp_fuid\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ftp_last_auth_requested\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_community\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_bulk_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_get_responses\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_set_requests\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_display_string\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"snmp_up_si
nce\", \"type\":[\"null\",\"float\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_cipher\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_curve\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_server_name\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_resumed\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_next_protocol\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_established\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_cert_chain_fuids\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"tls_client_cert_chain_fuids\", \"type\":[\"null\
",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_version\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_auth_success\", \"type\":[\"null\",\"boolean\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_client\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_server\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_cipher_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_mac_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_compression_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_key_exchange_algorithm\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"ssh_host_key_algorithm\", \"type\":[\"null\",
\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_assigned_ip4\", \"type\":[\"null\",\"long\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_mac\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"dhcp_lease_time\", \"type\":[\"null\",\"double\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_user\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_nickname\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_command\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_value\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"irc_additional_data\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_in_packets\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n
{\"name\":\"flow_out_packets\", \"type\":[\"null\",\"int\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_conn_state\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_history\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_src_dscp\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_dst_dscp\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_input\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"flow_output\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_id\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_title\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_type\", \"type\":[\"null\",\"str
ing\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_status\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null},\n {\"name\":\"vuln_severity\", \"type\":[\"null\",\"string\"],\"doc\":\"TBD\", \"default\": null}\n ],\n \"doc\": \"A view schema for storing Apache Spot Event data.\"\n }"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.registerSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrlsForRegistration",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaRegistryUrls",
+ "value" : [ ]
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaLookupMode",
+ "value" : "SUBJECT"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subject",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.subjectToRegister",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.schemaId",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.avroCompression",
+ "value" : "NULL"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.binaryFieldPath",
+ "value" : "/"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.protoDescriptorFile",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.messageType",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.fileNameEL",
+ "value" : null
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.wholeFileExistsAction",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.includeChecksumInTheEvents",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.checksumAlgorithm",
+ "value" : "MD5"
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlPrettyPrint",
+ "value" : true
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlValidateSchema",
+ "value" : false
+ }, {
+ "name" : "hdfsTargetConfigBean.dataGeneratorFormatConfig.xmlSchema",
+ "value" : null
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Hadoop FS 1",
+ "xPos" : 720,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ "FieldRemover_01OutputLane15059461007910" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ "HadoopFS_01_EventLane" ]
+ }, {
+ "instanceName" : "ExpressionEvaluator_02",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_processor_expression_ExpressionDProcessor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "expressionProcessorConfigs",
+ "value" : [ ]
+ }, {
+ "name" : "headerAttributeConfigs",
+ "value" : [ {
+ "attributeToSet" : "p_dvc_vendor",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -4), \".*=(.*)\", 1)}"
+ }, {
+ "attributeToSet" : "p_dvc_type",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -3), \".*=(.*)\", 1)}"
+ }, {
+ "attributeToSet" : "p_dt",
+ "headerAttributeExpression" : "${str:regExCapture(file:pathElement(record:value(\"/filepath\"), -2), \".*=(.*)\", 1)}"
+ } ]
+ }, {
+ "name" : "fieldAttributeConfigs",
+ "value" : [ {
+ "fieldToSet" : "/"
+ } ]
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Extract Partition Values",
+ "xPos" : 940,
+ "yPos" : 200,
+ "stageType" : "PROCESSOR"
+ },
+ "inputLanes" : [ "HadoopFS_01_EventLane" ],
+ "outputLanes" : [ "ExpressionEvaluator_02OutputLane15059713441990" ],
+ "eventLanes" : [ ]
+ }, {
+ "instanceName" : "HiveQuery_01",
+ "library" : "streamsets-datacollector-cdh_5_11-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_hive_queryexecutor_HiveQueryDExecutor",
+ "stageVersion" : "2",
+ "configuration" : [ {
+ "name" : "config.hiveConfigBean.hiveJDBCUrl",
+ "value" : "${HIVE_URL}"
+ }, {
+ "name" : "config.hiveConfigBean.hiveJDBCDriver",
+ "value" : "org.apache.hive.jdbc.HiveDriver"
+ }, {
+ "name" : "config.hiveConfigBean.confDir",
+ "value" : "hive-conf"
+ }, {
+ "name" : "config.hiveConfigBean.additionalConfigProperties",
+ "value" : [ ]
+ }, {
+ "name" : "config.queries",
+ "value" : [ "ALTER TABLE ${ODM_EVENTS_TABLE_NAME} ADD IF NOT EXISTS PARTITION (p_dvc_type='${record:attribute(\"p_dvc_type\")}', p_dvc_vendor='${record:attribute(\"p_dvc_vendor\")}', p_dt='${record:attribute(\"p_dt\")}')" ]
+ }, {
+ "name" : "config.stopOnQueryFailure",
+ "value" : true
+ }, {
+ "name" : "stageOnRecordError",
+ "value" : "TO_ERROR"
+ }, {
+ "name" : "stageRequiredFields",
+ "value" : [ ]
+ }, {
+ "name" : "stageRecordPreconditions",
+ "value" : [ ]
+ } ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Add Partition to Avro Table",
+ "xPos" : 1160,
+ "yPos" : 200,
+ "stageType" : "EXECUTOR"
+ },
+ "inputLanes" : [ "ExpressionEvaluator_02OutputLane15059713441990" ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "errorStage" : {
+ "instanceName" : "Discard_ErrorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Error Records - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "info" : {
+ "pipelineId" : "ODMWindowsEventLogs9fae6da4-eb7f-4aca-820a-1f561a40e7cc",
+ "title" : "ODMWindowsEventLogs",
+ "description" : "",
+ "created" : 1505971638546,
+ "lastModified" : 1505972055258,
+ "creator" : "natty@dpmfield",
+ "lastModifier" : "natty@dpmfield",
+ "lastRev" : "0",
+ "uuid" : "a155ab31-89d9-44d9-94d3-c06fc1fda992",
+ "valid" : true,
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "name" : "ODMWindowsEventLogs9fae6da4-eb7f-4aca-820a-1f561a40e7cc",
+ "sdcVersion" : "2.7.1.0",
+ "sdcId" : "f0ff6a12-61b1-44a6-bc5f-0d7e67d7d482"
+ },
+ "metadata" : {
+ "labels" : [ ]
+ },
+ "statsAggregatorStage" : {
+ "instanceName" : "WritetoDPMdirectly_StatsAggregatorStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_StatsDpmDirectlyDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stats Aggregator - Write to DPM directly",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ },
+ "startEventStages" : [ {
+ "instanceName" : "Discard_StartEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Start Event - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "stopEventStages" : [ {
+ "instanceName" : "Discard_StopEventStage",
+ "library" : "streamsets-datacollector-basic-lib",
+ "stageName" : "com_streamsets_pipeline_stage_destination_devnull_ToErrorNullDTarget",
+ "stageVersion" : "1",
+ "configuration" : [ ],
+ "uiInfo" : {
+ "description" : "",
+ "label" : "Stop Event - Discard",
+ "xPos" : 280,
+ "yPos" : 50,
+ "stageType" : "TARGET"
+ },
+ "inputLanes" : [ ],
+ "outputLanes" : [ ],
+ "eventLanes" : [ ]
+ } ],
+ "valid" : true,
+ "issues" : {
+ "issueCount" : 0,
+ "stageIssues" : { },
+ "pipelineIssues" : [ ]
+ },
+ "previewable" : true
+ },
+ "pipelineRules" : {
+ "schemaVersion" : 3,
+ "version" : 2,
+ "metricsRuleDefinitions" : [ {
+ "id" : "badRecordsAlertID",
+ "alertText" : "High incidence of Error Records",
+ "metricId" : "pipeline.batchErrorRecords.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1505945061638,
+ "valid" : true
+ }, {
+ "id" : "stageErrorAlertID",
+ "alertText" : "High incidence of Stage Errors",
+ "metricId" : "pipeline.batchErrorMessages.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > 100}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1505945061638,
+ "valid" : true
+ }, {
+ "id" : "idleGaugeID",
+ "alertText" : "Pipeline is Idle",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "TIME_OF_LAST_RECEIVED_RECORD",
+ "condition" : "${time:now() - value() > 120000}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1505945061638,
+ "valid" : true
+ }, {
+ "id" : "batchTimeAlertID",
+ "alertText" : "Batch taking more time to process",
+ "metricId" : "RuntimeStatsGauge.gauge",
+ "metricType" : "GAUGE",
+ "metricElement" : "CURRENT_BATCH_AGE",
+ "condition" : "${value() > 200}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1505945061638,
+ "valid" : true
+ }, {
+ "id" : "memoryLimitAlertID",
+ "alertText" : "Memory limit for pipeline exceeded",
+ "metricId" : "pipeline.memoryConsumed.counter",
+ "metricType" : "COUNTER",
+ "metricElement" : "COUNTER_COUNT",
+ "condition" : "${value() > (jvm:maxMemoryMB() * 0.65)}",
+ "sendEmail" : false,
+ "enabled" : false,
+ "timestamp" : 1505945061638,
+ "valid" : true
+ } ],
+ "dataRuleDefinitions" : [ ],
+ "driftRuleDefinitions" : [ ],
+ "uuid" : "dba0fd9a-f3b6-40fd-ada9-a87b8110ea65",
+ "configuration" : [ {
+ "name" : "emailIDs",
+ "value" : [ ]
+ }, {
+ "name" : "webhookConfigs",
+ "value" : [ ]
+ } ],
+ "ruleIssues" : [ ],
+ "configIssues" : [ ]
+ },
+ "libraryDefinitions" : null
+}
\ No newline at end of file