You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/06/02 09:45:06 UTC
[6/6] incubator-griffin git commit: remove streaming
remove streaming
Author: Liu <ll...@ebay.com>
Author: Lionel Liu <bh...@163.com>
Author: Liu <ll...@lm-shc-16501428.corp.ebay.com>
Author: Liu <ll...@lm-shc-16501428.dhcp>
Author: bhlx3lyx7 <bh...@163.com>
Closes #55 from bhlx3lyx7/master.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/a5d54703
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/a5d54703
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/a5d54703
Branch: refs/heads/master
Commit: a5d54703946072f0a95c3035ff293116092b8439
Parents: d3f21b1
Author: Liu <ll...@ebay.com>
Authored: Fri Jun 2 17:44:52 2017 +0800
Committer: Lionel Liu <bh...@163.com>
Committed: Fri Jun 2 17:44:52 2017 +0800
----------------------------------------------------------------------
.../dependency-reduced-pom.xml | 319 --------
griffin-streaming-model/pom.xml | 187 -----
.../src/main/resources/config.json | 190 -----
.../resources/kafkaConsumerConfig.properties | 2 -
.../scala/org/apache/griffin/Application.scala | 70 --
.../org/apache/griffin/algo/AccuracyAlgo.scala | 11 -
.../scala/org/apache/griffin/algo/Algo.scala | 11 -
.../griffin/algo/StreamingAccuracyAlgo.scala | 27 -
.../algo/StreamingAccuracyAlgo4Crawler.scala | 770 -------------------
.../apache/griffin/algo/core/AccuracyCore.scala | 92 ---
.../apache/griffin/cache/CacheProcesser.scala | 54 --
.../apache/griffin/cache/CachedAccuData.scala | 23 -
.../org/apache/griffin/cache/CachedData.scala | 27 -
.../griffin/config/ConfigFileReader.scala | 20 -
.../apache/griffin/config/ConfigReader.scala | 11 -
.../griffin/config/params/AccuracyParam.scala | 11 -
.../apache/griffin/config/params/AllParam.scala | 15 -
.../griffin/config/params/ConfigParam.scala | 10 -
.../griffin/config/params/DataAssetParam.scala | 14 -
.../griffin/config/params/DqConfigParam.scala | 10 -
.../apache/griffin/config/params/DqType.scala | 11 -
.../griffin/config/params/DumpParam.scala | 14 -
.../griffin/config/params/MappingParam.scala | 12 -
.../config/params/ParseConfigParam.scala | 10 -
.../griffin/config/params/ParseParam.scala | 12 -
.../griffin/config/params/PrepParam.scala | 11 -
.../griffin/config/params/RecorderParam.scala | 12 -
.../griffin/config/params/RetryParam.scala | 12 -
.../griffin/config/params/SampleParam.scala | 11 -
.../config/params/SchemaFieldParam.scala | 20 -
.../griffin/config/params/SparkParam.scala | 15 -
.../griffin/config/params/TimeRangeParam.scala | 12 -
.../griffin/config/params/TypeConfigParam.scala | 11 -
.../org/apache/griffin/dump/DumpWrapper.scala | 21 -
.../apache/griffin/dump/DumpWrapperInfo.scala | 32 -
.../apache/griffin/dump/HdfsFileDumpUtil.scala | 37 -
.../apache/griffin/prep/parse/DataParser.scala | 10 -
.../griffin/prep/parse/Json2MapParser.scala | 59 --
.../apache/griffin/record/HdfsRecorder.scala | 87 ---
.../apache/griffin/record/NullRecorder.scala | 20 -
.../apache/griffin/record/PostRecorder.scala | 46 --
.../org/apache/griffin/record/Recorder.scala | 19 -
.../apache/griffin/record/RecorderFactory.scala | 48 --
.../griffin/record/cleaner/CleanerFactory.scala | 31 -
.../griffin/record/cleaner/HdfsCleaner.scala | 78 --
.../griffin/record/result/AccuResult.scala | 18 -
.../apache/griffin/record/result/Result.scala | 12 -
.../org/apache/griffin/utils/DataTypeUtil.scala | 179 -----
.../apache/griffin/utils/ExtractJsonUtil.scala | 205 -----
.../org/apache/griffin/utils/HdfsUtil.scala | 57 --
.../org/apache/griffin/utils/JsonUtil.scala | 33 -
.../org/apache/griffin/utils/RestfulUtil.scala | 16 -
.../griffin/utils/ValueListCombineUtil.scala | 198 -----
.../src/test/resources/config.json | 11 -
.../src/test/resources/input.msg | 1 -
.../src/test/resources/out1.msg | 1 -
.../src/test/resources/output.msg | 1 -
.../griffin/config/ConfigFileReaderTest.scala | 21 -
.../apache/griffin/config/ConfigReadTest.scala | 45 --
.../griffin/prep/parse/Json2MapParserTest.scala | 49 --
.../org/apache/griffin/test/DataParseTest.scala | 101 ---
61 files changed, 3473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/dependency-reduced-pom.xml
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/dependency-reduced-pom.xml b/griffin-streaming-model/dependency-reduced-pom.xml
deleted file mode 100644
index 6fb4004..0000000
--- a/griffin-streaming-model/dependency-reduced-pom.xml
+++ /dev/null
@@ -1,319 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.griffin</groupId>
- <artifactId>griffin-crawler-model</artifactId>
- <version>1.0-SNAPSHOT</version>
- <build>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.2.2</version>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.4.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer>
- <mainClass>com.colobu.kafka.ConsumerExample</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
- <version>1.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>avro-mapred</artifactId>
- <groupId>org.apache.avro</groupId>
- </exclusion>
- <exclusion>
- <artifactId>chill_2.10</artifactId>
- <groupId>com.twitter</groupId>
- </exclusion>
- <exclusion>
- <artifactId>chill-java</artifactId>
- <groupId>com.twitter</groupId>
- </exclusion>
- <exclusion>
- <artifactId>xbean-asm5-shaded</artifactId>
- <groupId>org.apache.xbean</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-client</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spark-launcher_2.10</artifactId>
- <groupId>org.apache.spark</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spark-network-common_2.10</artifactId>
- <groupId>org.apache.spark</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spark-network-shuffle_2.10</artifactId>
- <groupId>org.apache.spark</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spark-unsafe_2.10</artifactId>
- <groupId>org.apache.spark</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jets3t</artifactId>
- <groupId>net.java.dev.jets3t</groupId>
- </exclusion>
- <exclusion>
- <artifactId>curator-recipes</artifactId>
- <groupId>org.apache.curator</groupId>
- </exclusion>
- <exclusion>
- <artifactId>javax.servlet</artifactId>
- <groupId>org.eclipse.jetty.orbit</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-lang3</artifactId>
- <groupId>org.apache.commons</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-math3</artifactId>
- <groupId>org.apache.commons</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jsr305</artifactId>
- <groupId>com.google.code.findbugs</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jul-to-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jcl-over-slf4j</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>compress-lzf</artifactId>
- <groupId>com.ning</groupId>
- </exclusion>
- <exclusion>
- <artifactId>RoaringBitmap</artifactId>
- <groupId>org.roaringbitmap</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-net</artifactId>
- <groupId>commons-net</groupId>
- </exclusion>
- <exclusion>
- <artifactId>akka-remote_2.10</artifactId>
- <groupId>com.typesafe.akka</groupId>
- </exclusion>
- <exclusion>
- <artifactId>akka-slf4j_2.10</artifactId>
- <groupId>com.typesafe.akka</groupId>
- </exclusion>
- <exclusion>
- <artifactId>json4s-jackson_2.10</artifactId>
- <groupId>org.json4s</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-server</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jersey-core</artifactId>
- <groupId>com.sun.jersey</groupId>
- </exclusion>
- <exclusion>
- <artifactId>mesos</artifactId>
- <groupId>org.apache.mesos</groupId>
- </exclusion>
- <exclusion>
- <artifactId>netty-all</artifactId>
- <groupId>io.netty</groupId>
- </exclusion>
- <exclusion>
- <artifactId>stream</artifactId>
- <groupId>com.clearspring.analytics</groupId>
- </exclusion>
- <exclusion>
- <artifactId>metrics-core</artifactId>
- <groupId>io.dropwizard.metrics</groupId>
- </exclusion>
- <exclusion>
- <artifactId>metrics-jvm</artifactId>
- <groupId>io.dropwizard.metrics</groupId>
- </exclusion>
- <exclusion>
- <artifactId>metrics-json</artifactId>
- <groupId>io.dropwizard.metrics</groupId>
- </exclusion>
- <exclusion>
- <artifactId>metrics-graphite</artifactId>
- <groupId>io.dropwizard.metrics</groupId>
- </exclusion>
- <exclusion>
- <artifactId>ivy</artifactId>
- <groupId>org.apache.ivy</groupId>
- </exclusion>
- <exclusion>
- <artifactId>oro</artifactId>
- <groupId>oro</groupId>
- </exclusion>
- <exclusion>
- <artifactId>tachyon-client</artifactId>
- <groupId>org.tachyonproject</groupId>
- </exclusion>
- <exclusion>
- <artifactId>pyrolite</artifactId>
- <groupId>net.razorvine</groupId>
- </exclusion>
- <exclusion>
- <artifactId>py4j</artifactId>
- <groupId>net.sf.py4j</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_2.10</artifactId>
- <version>1.6.0</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.10</artifactId>
- <version>1.6.0</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <artifactId>parquet-hadoop-bundle</artifactId>
- <groupId>com.twitter</groupId>
- </exclusion>
- <exclusion>
- <artifactId>spark-sql_2.10</artifactId>
- <groupId>org.apache.spark</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hive-exec</artifactId>
- <groupId>org.spark-project.hive</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hive-metastore</artifactId>
- <groupId>org.spark-project.hive</groupId>
- </exclusion>
- <exclusion>
- <artifactId>avro</artifactId>
- <groupId>org.apache.avro</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-httpclient</artifactId>
- <groupId>commons-httpclient</groupId>
- </exclusion>
- <exclusion>
- <artifactId>calcite-avatica</artifactId>
- <groupId>org.apache.calcite</groupId>
- </exclusion>
- <exclusion>
- <artifactId>calcite-core</artifactId>
- <groupId>org.apache.calcite</groupId>
- </exclusion>
- <exclusion>
- <artifactId>httpclient</artifactId>
- <groupId>org.apache.httpcomponents</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jackson-mapper-asl</artifactId>
- <groupId>org.codehaus.jackson</groupId>
- </exclusion>
- <exclusion>
- <artifactId>commons-codec</artifactId>
- <groupId>commons-codec</groupId>
- </exclusion>
- <exclusion>
- <artifactId>joda-time</artifactId>
- <groupId>joda-time</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jodd-core</artifactId>
- <groupId>org.jodd</groupId>
- </exclusion>
- <exclusion>
- <artifactId>datanucleus-core</artifactId>
- <groupId>org.datanucleus</groupId>
- </exclusion>
- <exclusion>
- <artifactId>libthrift</artifactId>
- <groupId>org.apache.thrift</groupId>
- </exclusion>
- <exclusion>
- <artifactId>libfb303</artifactId>
- <groupId>org.apache.thrift</groupId>
- </exclusion>
- <exclusion>
- <artifactId>avro-mapred</artifactId>
- <groupId>org.apache.avro</groupId>
- </exclusion>
- <exclusion>
- <artifactId>jsr305</artifactId>
- <groupId>com.google.code.findbugs</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <artifactId>hamcrest-core</artifactId>
- <groupId>org.hamcrest</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_2.10</artifactId>
- <version>2.2.4</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <properties>
- <avro.version>1.7.7</avro.version>
- <scala.binary.version>2.10</scala.binary.version>
- <jackson.version>2.8.7</jackson.version>
- <maven.compiler.source>1.8</maven.compiler.source>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <spark.version>1.6.0</spark.version>
- <maven.compiler.target>1.8</maven.compiler.target>
- </properties>
-</project>
-
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/pom.xml
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/pom.xml b/griffin-streaming-model/pom.xml
deleted file mode 100644
index ac96542..0000000
--- a/griffin-streaming-model/pom.xml
+++ /dev/null
@@ -1,187 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.griffin</groupId>
- <artifactId>griffin-crawler-model</artifactId>
- <version>1.0-SNAPSHOT</version>
-
-
- <properties>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-
- <maven.compiler.source>1.8</maven.compiler.source>
- <maven.compiler.target>1.8</maven.compiler.target>
-
- <spark.version>1.6.0</spark.version>
- <scala.binary.version>2.10</scala.binary.version>
- <avro.version>1.7.7</avro.version>
-
- <jackson.version>2.8.7</jackson.version>
- <!--<javax.validation.version>1.1.0.Final</javax.validation.version>-->
- <!--<hibernate.validator.version>5.4.0.Final</hibernate.validator.version>-->
- <!--<javax.el.version>2.2.4</javax.el.version>-->
-
- </properties>
-
- <dependencies>
- <!--<dependency>-->
- <!--<groupId>org.apache.kafka</groupId>-->
- <!--<artifactId>kafka_${scala.binary.version}</artifactId>-->
- <!--<version>0.8.2.1</version>-->
- <!--<exclusions>-->
- <!--<exclusion>-->
- <!--<artifactId>jmxri</artifactId>-->
- <!--<groupId>com.sun.jmx</groupId>-->
- <!--</exclusion>-->
- <!--<exclusion>-->
- <!--<artifactId>jmxtools</artifactId>-->
- <!--<groupId>com.sun.jdmk</groupId>-->
- <!--</exclusion>-->
- <!--<exclusion>-->
- <!--<artifactId>jms</artifactId>-->
- <!--<groupId>javax.jms</groupId>-->
- <!--</exclusion>-->
- <!--<exclusion>-->
- <!--<artifactId>junit</artifactId>-->
- <!--<groupId>junit</groupId>-->
- <!--</exclusion>-->
- <!--</exclusions>-->
- <!--</dependency>-->
- <!--<dependency>-->
- <!--<groupId>junit</groupId>-->
- <!--<artifactId>junit</artifactId>-->
- <!--<version>3.8.1</version>-->
- <!--<scope>test</scope>-->
- <!--</dependency>-->
-
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <!--<dependency>-->
- <!--<groupId>org.apache.spark</groupId>-->
- <!--<artifactId>spark-streaming-kafka-0-10_${scala.binary.version}</artifactId>-->
- <!--<version>${spark.version}</version>-->
- <!--</dependency>-->
-
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kafka_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${jackson.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.module</groupId>
- <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- <version>${jackson.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.scalaj</groupId>
- <artifactId>scalaj-http_${scala.binary.version}</artifactId>
- <version>2.3.0</version>
- </dependency>
-
- <!--<dependency>-->
- <!--<groupId>javax.validation</groupId>-->
- <!--<artifactId>validation-api</artifactId>-->
- <!--<version>${javax.validation.version}</version>-->
- <!--</dependency>-->
- <!--<dependency>-->
- <!--<groupId>org.hibernate</groupId>-->
- <!--<artifactId>hibernate-validator</artifactId>-->
- <!--<version>${hibernate.validator.version}</version>-->
- <!--</dependency>-->
- <!--<dependency>-->
- <!--<groupId>javax.el</groupId>-->
- <!--<artifactId>javax.el-api</artifactId>-->
- <!--<version>${javax.el.version}</version>-->
- <!--</dependency>-->
- <!--<dependency>-->
- <!--<groupId>org.glassfish.web</groupId>-->
- <!--<artifactId>javax.el</artifactId>-->
- <!--<version>${javax.el.version}</version>-->
- <!--</dependency>-->
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.11</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.binary.version}</artifactId>
- <version>2.2.4</version>
- <scope>test</scope>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.2.2</version>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.4.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer
- implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>com.colobu.kafka.ConsumerExample</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-
-</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/resources/config.json
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/resources/config.json b/griffin-streaming-model/src/main/resources/config.json
deleted file mode 100644
index eaa6452..0000000
--- a/griffin-streaming-model/src/main/resources/config.json
+++ /dev/null
@@ -1,190 +0,0 @@
-{
- "spark": {
- "app.name": "GriffinAccuStreamingApp",
- "log.level": "INFO",
- "config": {
- "spark.task.maxFailures": 5,
- "spark.streaming.kafka.maxRatePerPartition": 1000,
- "spark.streaming.concurrentJobs": 4
- },
- "streaming.checkpoint.dir": "hdfs:///test/kafka/cp",
- "streaming.batch.interval.seconds": 5,
- "streaming.sample.interval.seconds": 5
- },
-
- "type.config": {
- "dq": "accuracy",
- "dataAsset": {
- "source": "kafka",
- "target": "kafka"
- }
- },
-
- "dataAsset": {
- "source": {
- "kafka.config": {
- "bootstrap.servers": "10.9.246.187:9092",
- "group.id": "group1",
- "auto.offset.reset": "smallest",
- "auto.commit.enable": "false"
- },
- "topics": "crwaler-in",
- "pre.process": {
- "sample": {
- "need.sample": true,
- "ratio": 1
- },
- "parse": {
- "in.type": "json",
- "out.type": "map",
- "config": {
- "out.schema": [
- {
- "name": "url",
- "type": "string",
- "default.value": "",
- "extract.steps": ["json", ".seeds", "[*]", "json", ".url"]
- },
- {
- "name": "createdts",
- "type": "bigint",
- "default.value": "0",
- "extract.steps": ["json", ".seeds", "[*]", "json", ".metadata", "json", ".tracker", ".crawlRequestCreateTS"]
- }
- ]
- }
- }
- },
- "dump.config": {
- "table.name": "cin",
- "dir": "hdfs:///griffin/streaming/dump",
- "schema": [
- {
- "name": "url",
- "type": "string",
- "default.value": ""
- },
- {
- "name": "createdts",
- "type": "bigint",
- "default.value": "0"
- }
- ]
- },
- "schema": [
- {
- "name": "url",
- "type": "string",
- "default.value": ""
- },
- {
- "name": "createdts",
- "type": "bigint",
- "default.value": "0"
- }
- ]
- },
- "target": {
- "kafka.config": {
- "bootstrap.servers": "10.9.246.187:9092",
- "group.id": "group1",
- "auto.offset.reset": "smallest",
- "auto.commit.enable": "false"
- },
- "topics": "crawler-out",
- "pre.process": {
- "parse": {
- "in.type": "json",
- "out.type": "map",
- "config": {
- "out.schema": [
- {
- "name": "url",
- "type": "string",
- "default.value": "",
- "extract.steps": ["json", ".groups", "[0]", ".attrsList", "[.name=URL]", ".values", "[0]"]
- },
- {
- "name": "createdts",
- "type": "bigint",
- "default.value": "0",
- "extract.steps": ["json", ".groups", "[0]", ".attrsList", "[.name=CRAWLMETADATA]", ".values", "[0]", "json", ".tracker", ".crawlRequestCreateTS"]
- }
- ]
- }
- }
- },
- "dump.config": {
- "table.name": "crawlerout",
- "dir": "hdfs:///griffin/streaming/dump",
- "schema": [
- {
- "name": "url",
- "type": "string",
- "default.value": ""
- },
- {
- "name": "createdts",
- "type": "bigint",
- "default.value": "0"
- }
- ]
- },
- "schema": [
- {
- "name": "url",
- "type": "string",
- "default.value": ""
- },
- {
- "name": "createdts",
- "type": "bigint",
- "default.value": "0"
- }
- ]
- }
- },
-
- "dq.config": {
- "accuracy": {
- "match.valid.time": {
- "begin": 0,
- "end": 24,
- "unit": "hour"
- },
- "mapping": [
- {
- "source.name": "url",
- "target.name": "url",
- "isPK": true
- }, {
- "source.name": "createdts",
- "target.name": "createdts",
- "isPK": true
- }
- ]
- }
- },
-
- "recorder": {
- "types": ["hdfs", "post"],
- "metric.name": "nrt_accuracy_crawler",
- "config": {
- "hdfs.dir": "hdfs:///griffin/streaming/record/",
- "post.url": "http://phxbark4dq-360935.stratus.phx.ebay.com:8080/",
- "post.metric.name": "nrt_accuracy_crawler"
- }
- },
-
- "cleaner": {
-
- },
-
- "retry": {
- "need.retry": true,
- "next.retry.seconds": 300,
- "interval.seconds": 60
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/resources/kafkaConsumerConfig.properties
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/resources/kafkaConsumerConfig.properties b/griffin-streaming-model/src/main/resources/kafkaConsumerConfig.properties
deleted file mode 100644
index b54b5e4..0000000
--- a/griffin-streaming-model/src/main/resources/kafkaConsumerConfig.properties
+++ /dev/null
@@ -1,2 +0,0 @@
-bootstrap.servers = 10.9.246.187:9092
-group.id =
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/Application.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/Application.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/Application.scala
deleted file mode 100644
index 38247eb..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/Application.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-package org.apache.griffin
-
-import org.apache.griffin.algo._
-import org.apache.griffin.config.ConfigFileReader
-import org.apache.griffin.config.params._
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-import org.apache.spark.{Logging, SparkConf, SparkContext}
-
-import scala.collection.JavaConverters._
-
-object Application extends Logging {
-//object Application {
-
- def main(args: Array[String]): Unit = {
- if (args.length < 1) {
- logError("Usage: class [<config-files>]")
- sys.exit(-1)
- }
-
- val Array(configFile) = args
-
- // read config file
- val configReader = ConfigFileReader(configFile)
-// val allParam = configReader.readConfig[AllParam]
- val allParam = configReader.readConfig
-
- // spark config
- val sparkParam = allParam.sparkParam
-
- // type config
- val typeConfigParam = allParam.typeConfigParam
- val dqType = DqType.parse(typeConfigParam.dqType)
- val dataAssetTypeMap = typeConfigParam.dataAssetTypeMap
-
- // data assets
- val dataAssetParamMap = allParam.dataAssetParamMap
-
- // validation of params for algo
- val validation = dqType match {
- case AccuracyType => {
- (dataAssetTypeMap.contains("source") && dataAssetParamMap.contains("source")) &&
- (dataAssetTypeMap.contains("target") && dataAssetParamMap.contains("target"))
- }
- case _ => (dataAssetTypeMap.contains("source") && dataAssetParamMap.contains("source"))
- }
- if (!validation) {
- logError("Config Error!")
- sys.exit(-1)
- }
-
- // choose algo
-// val algo: Algo = dqType match {
-// case AccuracyType => {
-// dataAssetTypeMap.get("source") match {
-// case Some("kafka") => StreamingAccuracyAlgo(sparkParam, dataAssetParamMap.get("source").get, dataAssetParamMap.get("target").get)
-// case _ => StreamingAccuracyAlgo(sparkParam, dataAssetParamMap.get("source").get, dataAssetParamMap.get("target").get)
-// }
-// }
-// case _ => {
-// StreamingAccuracyAlgo(sparkParam, dataAssetParamMap.get("source").get, dataAssetParamMap.get("target").get)
-// }
-// }
- val algo: Algo = StreamingAccuracyAlgo4Crawler(allParam)
-
- algo.run
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/AccuracyAlgo.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/AccuracyAlgo.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/AccuracyAlgo.scala
deleted file mode 100644
index f09be1b..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/AccuracyAlgo.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.algo
-
-import org.apache.griffin.config.params._
-
-
-trait AccuracyAlgo extends Algo {
-
- val sourceDataParam: DataAssetParam
- val targetDataParam: DataAssetParam
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/Algo.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/Algo.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/Algo.scala
deleted file mode 100644
index d7f51b3..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/Algo.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.algo
-
-import org.apache.griffin.config.params._
-
-trait Algo extends Serializable {
-
- val sparkParam: SparkParam
-
- def run(): Unit
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo.scala
deleted file mode 100644
index f0f2023..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.apache.griffin.algo
-
-import org.apache.griffin.config.params._
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-import org.apache.spark.{SparkConf, SparkContext}
-
-case class StreamingAccuracyAlgo(val sparkParam: SparkParam,
- val sourceDataParam: DataAssetParam,
- val targetDataParam: DataAssetParam
- ) extends AccuracyAlgo {
-
- def run(): Unit = {
- // create context
-// val sparkConf = new SparkConf().setAppName(sparkParam.appName)
-// sparkConf.setAll(sparkParam.config)
-//
-// val sc = new SparkContext(sparkConf)
-// sc.setLogLevel(sparkParam.logLevel)
-// val sqlContext = new HiveContext(sc)
-// val ssc = new StreamingContext(sc, Seconds(sparkParam.interval))
-// ssc.checkpoint(sparkParam.cpDir)
-
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo4Crawler.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo4Crawler.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo4Crawler.scala
deleted file mode 100644
index caa7ad5..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/StreamingAccuracyAlgo4Crawler.scala
+++ /dev/null
@@ -1,770 +0,0 @@
-package org.apache.griffin.algo
-
-import java.util.Date
-
-import kafka.serializer.StringDecoder
-import org.apache.griffin.algo.core.AccuracyCore
-import org.apache.griffin.cache.{CacheProcesser, CachedAccuData}
-import org.apache.griffin.config.params._
-import org.apache.griffin.dump._
-import org.apache.griffin.prep.parse.{DataParser, Json2MapParser}
-import org.apache.griffin.record.{Recorder, RecorderFactory}
-import org.apache.griffin.utils.HdfsUtil
-import org.apache.spark.rdd.{RDD, EmptyRDD}
-import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.streaming.dstream.InputDStream
-import org.apache.spark.streaming.kafka.KafkaUtils
-import org.apache.spark.streaming.{Seconds, StreamingContext}
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.griffin.record.result.{AccuResult, Result}
-
-
-case class StreamingAccuracyAlgo4Crawler( val allParam: AllParam
- ) extends AccuracyAlgo {
-
- val sparkParam: SparkParam = allParam.sparkParam
- val sourceDataParam: DataAssetParam = allParam.dataAssetParamMap.get("source").get
- val targetDataParam: DataAssetParam = allParam.dataAssetParamMap.get("target").get
-
- // current dumping hour and minute
- private var dumpMin: Long = 0L
-
- // current checked minute
- private var checkMin: Long = 0L
-
- // next fire time
- private var nextFireTime: Long = 0L
-
-
-
- def run(): Unit = {
- val ssc = StreamingContext.getOrCreate(sparkParam.cpDir,
- ( ) => {
- try {
- createContext()
- } catch {
- case runtime: RuntimeException => {
- throw runtime
- }
- }
- })
-
- ssc.start()
- ssc.awaitTermination()
- ssc.stop(stopSparkContext=true, stopGracefully=true)
- }
-
- def createContext(): StreamingContext = {
- // create context
- val sparkConf = new SparkConf().setAppName(sparkParam.appName)
- sparkConf.setAll(sparkParam.config)
-
- val sc = new SparkContext(sparkConf)
- sc.setLogLevel(sparkParam.logLevel)
- val sqlContext = new HiveContext(sc)
- val interval = Seconds(sparkParam.batchInterval)
- val ssc = new StreamingContext(sc, interval)
- ssc.checkpoint(sparkParam.cpDir)
-
- // emptyRDD
- val emptyRdd = sc.emptyRDD[(Product, (Map[String, Any], Map[String, Any]))]
-
- // source data direct stream
- val sourceDStream = createDirectStream(ssc, sourceDataParam)
-
- // target data direct stream
- val targetDStream = createDirectStream(ssc, targetDataParam)
-
- // accuracy param
- val accuracyParam = allParam.dqConfigParam.accuracy
- val matchValidTime = accuracyParam.matchValidTime
- val timeRange = (getTimeMs(matchValidTime.begin, matchValidTime.unit), getTimeMs(matchValidTime.end, matchValidTime.unit))
-
- val mapping = accuracyParam.mapping
- // get mapping Keys and Values
- val mappingKey = mapping.filter(_.isPK)
- val mappingValue = mapping
-
- // retry param
- val retryParam = allParam.retryParam
- val nextRetryInterval = retryParam.nextRetryInterval * 1000
- val retryPollInterval = retryParam.interval * 1000
- val retryInterval = Seconds(retryParam.interval)
- val cacheProcesser = CacheProcesser(retryParam)
-
- // recorder
- val recorderFactory = RecorderFactory(allParam.recorderParam)
-
- //--- 1. target part ---
-
- // target data from kafka (crawler output)
- val targetDumpConfigParam = targetDataParam.dumpConfigParam
- val targetPrepParam = targetDataParam.prepParam
- val targetSchema = targetDataParam.schema
-
- // target parser
- val targetParser = Json2MapParser(targetPrepParam.parseParam.configParam)
-
- // create hive table for target data
- val targetDumpWrapper = DumpWrapper(List[DumpWrapperInfo]())
- val targetPartitionDef = List[(String, String)](("hr", "bigint"), ("min", "bigint"))
- val (targetCreateTableSql, targetDumpDir) = genCreateTableSql(targetDumpConfigParam, targetDumpWrapper, targetPartitionDef)
- val targetTableName = targetDumpConfigParam.tableName
-// val fieldSep = targetDumpConfigParam.fieldSep
-// val lineSep = targetDumpConfigParam.lineSep
- val targetFieldSep = ","
- val targetLineSep = "\n"
- sqlContext.sql(targetCreateTableSql)
-
- // target dstream dump process
- targetDStream.foreachRDD((rdd, time) => {
- val ms = time.milliseconds
- val min = getTimeToUnit(ms, "min")
- val hour = getTimeToUnit(ms, "hour")
- val partitions = List[(String, Any)](("hr", hour), ("min", min))
-
- val partitionPath = genPartitionHdfsPath(partitions)
- val path = s"${targetDumpDir}/${partitionPath}/${ms}"
-
- // parse each message
- val recordRdd: RDD[String] = rdd.flatMap { kv =>
- val msg = kv._2
-
- // parse rdd msg
- val res: Seq[Map[String, Any]] = targetParser.parse(msg)
-
- // the sequence is the same with schema sequence, no need to sort here
- val records = res.map { r =>
- r.values.mkString(targetFieldSep)
- }
-
- records
- }
-
- if (!recordRdd.isEmpty) {
-
- // save to hdfs
- HdfsFileDumpUtil.dump(path, recordRdd, targetLineSep)
-
- // update hive
- val partitionSql = genAddPartitionSql(targetTableName, partitions)
- sqlContext.sql(partitionSql)
- }
-
- // update dumpHourMin
- updateDumpMin(min)
- })
-
- //--- 2. source part ---
-
- // source data from kafka (crawler input)
- val sourceDumpConfigParam = sourceDataParam.dumpConfigParam
- val sourcePrepParam = sourceDataParam.prepParam
- val sourceSchema = sourceDataParam.schema
- val sourceSampleParam = sourcePrepParam.sampleParam
-
- // source parser
- val sourceParser = Json2MapParser(sourcePrepParam.parseParam.configParam)
-
- // create hive table for source data
- val sourceDumpWrapper = DumpWrapper(List[DumpWrapperInfo](TimeGroupInfo, NextFireTimeInfo, MismatchInfo, ErrorInfo))
- val sourcePartitionDef = List[(String, String)](("firetime", "bigint"))
- val (sourceCreateTableSql, sourceDumpDir) = genCreateTableSql(sourceDumpConfigParam, sourceDumpWrapper, sourcePartitionDef)
- val sourceTableName = sourceDumpConfigParam.tableName
- val sourceFieldSep = ","
- val sourceLineSep = "\n"
- sqlContext.sql(sourceCreateTableSql)
-
- // get source dump keys
- val sourceDumpValueKeys = sourceDumpConfigParam.schema.map(_.name)
- val sourceDumpInfoKeys = sourceDumpWrapper.wrapKeys
-
- // source dstream parse and compare process
- val sampleInterval = Seconds(sparkParam.sampleInterval)
- sourceDStream.foreachRDD((rdd, time) => {
- val ms = time.milliseconds
-
- val batchTime = new Date()
- println(s"++++++ ${batchTime} ++++++")
-
- val curMin = getTimeToUnit(ms, "min")
-
- // get next fire time
- val cft = getNextFireTime
- val (nft, fireNow, updateHive) = if (cft <= 0) {
- initNextFireTime(ms, nextRetryInterval, retryPollInterval)
- (getNextFireTime, false, true)
- } else if (cft <= ms) {
- updateNextFireTime(nextRetryInterval)
- (getNextFireTime, true, true)
- } else (cft, false, false)
-
- // get check min
- val icm = getCheckMin
- val curCheckMin = if (icm <= 0) {
- updateCheckMin(curMin)
- getCheckMin
- } else icm
-
- // dump path
- val partitions = List[(String, Any)](("firetime", nft))
- val partitionPath = genPartitionHdfsPath(partitions)
- val dumpPath = s"${sourceDumpDir}/${partitionPath}/${ms}"
- val dumpRetryPath = s"${sourceDumpDir}/${partitionPath}/${ms}.r"
-
- // update hive
- if (updateHive) {
- val partitionSql = genAddPartitionSql(sourceTableName, partitions)
- sqlContext.sql(partitionSql)
- }
-
- // sample time process
- if (time.isMultipleOf(sampleInterval)) {
- val recorders = recorderFactory.getRecorders(ms)
-
- // start
- recorders.foreach(_.start)
-
- // sample data
- val sampleRdd = if (sourceSampleParam.needSample) {
- sampleFunc(rdd, sourceSampleParam.ratio)
- } else rdd
-
- val sampleCount = sampleRdd.count
- println(s"===== sampleCount: ${sampleCount} =====")
-
- // parse data
- if (sampleRdd.isEmpty) {
- // empty source data, do nothing here
- recorders.foreach(_.info(ms, "empty source data"))
- } else {
- // parse data
- val parseRdd: RDD[Map[String, Any]] = sampleRdd.flatMap { kv =>
- val msg = kv._2
- // parse msg
- sourceParser.parse(msg)
- }
-
- val parseCount = parseRdd.count
- println(s"===== parseCount: ${parseCount} =====")
-
- // get source data
- val sourceRdd: RDD[(Product, Map[String, Any])] = parseRdd.map { row =>
- val keys: List[AnyRef] = mappingKey.map { mp =>
- row.get(mp.sourceName) match {
- case Some(v) => v.asInstanceOf[AnyRef]
- case None => null
- }
- }
- val values: Map[String, Any] = mappingValue.map { mp =>
- val v = row.get(mp.sourceName) match {
- case Some(v) => v
- case None => null
- }
- (mp.sourceName, v)
- }.toMap
- (toTuple(keys) -> values)
- }
- val sourceCount = sourceRdd.count
-
- println(s"===== first time sourceCount: ${sourceCount} =====")
-
- // wrap source data
- val sourceWrappedRdd: RDD[(Product, (Map[String, Any], Map[String, Any]))] = sourceRdd.map { r =>
- val wrappedData = Map[String, Any](
- (TimeGroupInfo.key -> ms),
- (NextFireTimeInfo.key -> nft),
- (MismatchInfo.key -> ""),
- (ErrorInfo.key -> "")
- )
- (r._1, (r._2, wrappedData))
- }
-
- val (result, missRdd, matchRdd) = accu(curMin, sourceWrappedRdd)
-
- val missingRddCount1 = missRdd.count
- println(s"===== missingRddCount1: ${missingRddCount1} =====")
-
- // record result
- recorders.foreach(_.accuracyResult(ms, result))
-
- val missStrings = missRdd.map { row =>
- val (key, (value, info)) = row
- s"${value} [${info.getOrElse(MismatchInfo.key, "unknown")}]"
- }.collect()
-
- // record missing records
- recorders.foreach(_.accuracyMissingRecords(missStrings))
-
- // cache
- val cacheData = CachedAccuData()
- cacheData.result = result
- cacheData.curTime = ms
- cacheData.deadTime = timeRange._2 + ms
- cacheProcesser.cache(ms, cacheData)
-
- // dump mismatch data to hive
- if (!missRdd.isEmpty) {
- // parse each record
- val dumpMissingRdd: RDD[String] = missRdd.map { kv =>
- val (_, (dt, inf)) = kv
- val values = getDumpValues(dt, sourceDumpValueKeys)
- val infos = getDumpValues(inf, sourceDumpInfoKeys)
- (values ::: infos).mkString(sourceFieldSep)
- }
-
- // save to hdfs
- HdfsFileDumpUtil.dump(dumpPath, dumpMissingRdd, sourceLineSep)
-
- }
-
- }
-
- // finish
- recorders.foreach(_.finish)
-
- }
-
- // retry
- if (fireNow) {
-
- val t1 = new Date().getTime
- println("===== retrying begin =====")
-
- // get current path
- val partitions = List[(String, Any)](("firetime", cft))
- val partitionPath = genPartitionHdfsPath(partitions)
- val dumpedDirPath = s"${sourceDumpDir}/${partitionPath}"
-
- println(s"===== nft: ${nft} =====")
- println(s"===== cft: ${cft} =====")
- println(s"===== dumpedDirPath: ${dumpedDirPath} =====")
-
- val tt1 = new Date().getTime
-
- // get missing data from hive
- val partitionsRange = List(("firetime", cft))
- val selectSql = genSelectSql(sourceTableName, partitionsRange)
- val missingDataFrame: DataFrame = sqlContext.sql(selectSql)
-
- val tt2 = new Date().getTime
- println(s"--------------- get source data from hive time: ${tt2 - tt1} ---------------")
-
- // get missing data
- val missingSourceWrappedRdd: RDD[(Product, (Map[String, Any], Map[String, Any]))] = missingDataFrame.map { row =>
- val keys: List[AnyRef] = mappingKey.map { mp =>
- row.getAs[Any](mp.sourceName).asInstanceOf[AnyRef]
- }
- val values: Map[String, Any] = sourceDumpValueKeys.map { key =>
- val v = row.getAs[Any](key)
- (key, v)
- }.toMap
- val infos: Map[String, Any] = sourceDumpInfoKeys.map { key =>
- val v = row.getAs[Any](key)
- (key, v)
- }.toMap
- (toTuple(keys) -> (values, infos))
- }
-
- val tt3 = new Date().getTime
- println(s"--------------- wrap source data time: ${tt3 - tt2} ---------------")
-
- if (!missingSourceWrappedRdd.isEmpty) {
- val sourceCount = missingSourceWrappedRdd.count
- println(s"===== rerty sourceCount: ${sourceCount} =====")
-
- val (retryResult, stillMissingRdd, matchRdd) = accu(curCheckMin, missingSourceWrappedRdd)
-
- val tt4 = new Date().getTime
- println(s"--------------- accu time: ${tt4 - tt3} ---------------")
-
- val missingRddCount2 = stillMissingRdd.count
- println(s"===== missingRddCount2: ${missingRddCount2} =====")
-
- val matchRddCount2 = matchRdd.count
- println(s"===== matchRddCount2: ${matchRddCount2} =====")
-
- // filter still need saving rdd
- val savingRdd: RDD[(Product, (Map[String, Any], Map[String, Any]))] = stillMissingRdd.filter { row =>
- val (key, (value, info)) = row
- info.get(TimeGroupInfo.key) match {
- case Some(t: Long) => {
- cacheProcesser.getCache(t) match {
- case Some(cache) => true
- case _ => false
- }
- }
- case _ => false
- }
- }
-
- def reorgByTimeGroup(rdd: RDD[(Product, (Map[String, Any], Map[String, Any]))]
- ): RDD[(Long, (Product, (Map[String, Any], Map[String, Any])))] = {
- rdd.flatMap { row =>
- val (key, (value, info)) = row
- val b: Option[(Long, (Product, (Map[String, Any], Map[String, Any])))] = info.get(TimeGroupInfo.key) match {
- case Some(t: Long) => Some((t, row))
- case _ => None
- }
- b
- }
- }
-
- // get missing results
- val missingResults = reorgByTimeGroup(stillMissingRdd)
- val missingResultsCount = missingResults.count
- println(s"===== missingResultsCount: ${missingResultsCount} =====")
-
- // get matched results
- val matchResults = reorgByTimeGroup(matchRdd)
- val matchResultsCount = matchResults.count
- println(s"===== matchResultsCount: ${matchResultsCount} =====")
-
- val groupedResults = missingResults.cogroup(matchResults)
-
- val tt5 = new Date().getTime
- println(s"--------------- cogroup time: ${tt5 - tt4} ---------------")
-
-// val groupedMatchResults: RDD[(Long, Iterable[(Product, (Map[String, Any], Map[String, Any]))])] = matchResultsCount.groupByKey()
-
- val groupedResultsCount = groupedResults.count
- println(s"===== groupedResultsCount: ${groupedResultsCount} =====")
-
- groupedResults.foreach(row => println(s"=== ${row._1}: ${row._2._1.size}, ${row._2._2.size} === ${ms} ==="))
-
- val retry_time = new Date().getTime
-
- // record
- groupedResults.foreach { rslt =>
- val (t, (missRes, matchRes)) = rslt
- val curRecorders = recorderFactory.getRecorders(t)
-
- cacheProcesser.getCache(t) match {
- case Some(cache) => {
- val matchCount = matchRes.size
- // need to update result
- if (matchCount > 0) {
- val accuCache = cache.asInstanceOf[CachedAccuData]
- val delta = AccuResult(missRes.size, (missRes.size + matchRes.size))
- val oldResult = accuCache.result
- val newResult = accuCache.result.updateResult(delta)
-
- // different result
- if (newResult.differsFrom(oldResult)) {
- accuCache.result = newResult
-
- // record result
- curRecorders.foreach(_.accuracyResult(ms, newResult))
-
- val missStrings = missRes.map { row =>
- val (key, (value, info)) = row
- s"${value} [${info.getOrElse(MismatchInfo.key, "unknown")}]"
- }
- // record missing records
- curRecorders.foreach(_.accuracyMissingRecords(missStrings))
-
- curRecorders.foreach(_.recordTime(ms, retry_time - t1))
- println(s"----- record here: [${ms}, (${t}: ${newResult})] -----")
- }
- }
- }
- case _ => {
- println(s"=== no cache of ${t}, need to clear ===")
- }
- }
- }
-
- val savingRddCount = savingRdd.count
- println(s"===== savingRddCount: ${savingRddCount} =====")
-
- val tt6 = new Date().getTime
- println(s"--------------- record and get saving data time: ${tt6 - tt5} ---------------")
-
- // caches refresh
- cacheProcesser.refresh(ms)
-
- // dump mismatch data to hive
- if (!savingRdd.isEmpty) {
- // parse each record
- val dumpMissingRdd: RDD[String] = savingRdd.map { kv =>
- val (_, (dt, inf)) = kv
- val values = getDumpValues(dt, sourceDumpValueKeys)
- val infos = getDumpValues(inf, sourceDumpInfoKeys)
- (values ::: infos).mkString(sourceFieldSep)
- }
-
- // save to hdfs
- HdfsFileDumpUtil.dump(dumpRetryPath, dumpMissingRdd, sourceLineSep)
-
- val tt7 = new Date().getTime
- println(s"--------------- dump saving data time: ${tt7 - tt6} ---------------")
- }
- } else {
- println("===== retry source data empty =====")
- }
-
- val tt7 = new Date().getTime
-
- // remove current calculated dumped datas
- HdfsUtil.deleteHdfsPath(dumpedDirPath)
-
- val tt8 = new Date().getTime
- println(s"--------------- remove outtime data time: ${tt8 - tt7} ---------------")
-
- val t2 = new Date().getTime
- println(s"===== retrying end [using ${t2 - t1} ms] =====")
- }
-
-
- })
-
- def accu(curMin: Long, sourceWrappedRdd: RDD[(Product, (Map[String, Any], Map[String, Any]))]
- ): (AccuResult, RDD[(Product, (Map[String, Any], Map[String, Any]))], RDD[(Product, (Map[String, Any], Map[String, Any]))]) = {
- val t1 = new Date().getTime
-
- // get dump min
- val dumpedMin = getDumpedMin
-
- val mRange = (curMin, dumpedMin)
- println(s"===== mRange: ${mRange} =====")
-
- val ret = if (dumpReady(curMin, dumpedMin)) {
- // get target data from hive
- val minRange = (curMin, dumpedMin)
- val partitionsRange = List(("min", minRange))
- val selectSql = genSelectSql(targetTableName, partitionsRange)
- val targetDataFrame: DataFrame = sqlContext.sql(selectSql)
-
- // get target data
- val targetWrappedRdd: RDD[(Product, (Map[String, Any], Map[String, Any]))] = targetDataFrame.map { row =>
- val keys: List[AnyRef] = mappingKey.map { mp =>
- row.getAs[Any](mp.targetName).asInstanceOf[AnyRef]
- }
- val values: Map[String, Any] = mappingValue.map { mp =>
- val v = row.getAs[Any](mp.targetName)
- (mp.targetName, v)
- }.toMap
- (toTuple(keys) -> (values, Map[String, Any]()))
- }
-
- val targetCount = targetWrappedRdd.count
- println(s"===== targetCount: ${targetCount} =====")
-
- val (result, missRdd, matchRdd) = if (targetWrappedRdd.isEmpty) {
- val sourceCount = sourceWrappedRdd.count
- (AccuResult(sourceCount, sourceCount), sourceWrappedRdd, emptyRdd)
- } else {
- // cogroup source and target
- val allKvs = sourceWrappedRdd.cogroup(targetWrappedRdd)
-
- // accuracy algorithm calculation
- val (accuResult, missingRdd, matchingRdd) = AccuracyCore.accuracy(allKvs)
-
- println("source count: " + accuResult.totalCount + " missed count : " + accuResult.missCount)
-
- val missingCount = missingRdd.count
- println(s"===== missingCount: ${missingCount} =====")
-
- (accuResult, missingRdd, matchingRdd)
- }
-
- // update check min
- updateCheckMin(dumpedMin + 1)
-
- (result, missRdd, matchRdd)
-
- } else {
- println("===== dump data not ready =====")
- val sourceCount = sourceWrappedRdd.count
- (AccuResult(sourceCount, sourceCount), sourceWrappedRdd, emptyRdd)
- }
-
- val t2 = new Date().getTime
-
- println(s"===== accu using time ${t2 - t1} ms =====")
-
- ret
- }
-
- ssc
-
- }
-
- def createDirectStream(ssc: StreamingContext, dataAssetParam: DataAssetParam): InputDStream[(String, String)] = {
- val kafkaParam = dataAssetParam.kafkaConfig
- val topics = dataAssetParam.topics.split(",").toSet
- KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](
- ssc,
- kafkaParam,
- topics
- )
- }
-
- def genCreateTableSql(dumpConfigParam: DumpParam, dumpWrapper: DumpWrapper, partitionDef: List[(String, String)]): (String, String) = {
- val tableName = dumpConfigParam.tableName
- val targetDumpDir = s"${dumpConfigParam.dir}/${tableName}"
-
- val schema = dumpConfigParam.schema
- val schemaCols = schema.map { field =>
- (field.name, field.fieldType)
- }
-
- val wrapperCols = dumpWrapper.wrapSchema
-
- val colsSql = (schemaCols ++ wrapperCols).map(col => (s"`${col._1}` ${col._2}")).mkString(", ")
-
-// val partitions = dumpConfigParam.partitions
-// val partitionCols = partitions.map { field =>
-// (field.name, field.fieldType)
-// }.toMap
-// val partitionsSql = partitionCols.map(prtn => (s"`${prtn._1}` ${prtn._2}")).mkString(", ")
- val partitionsSql = partitionDef.map(prtn => (s"`${prtn._1}` ${prtn._2}")).mkString(", ")
-
- val sql =
- s"""CREATE EXTERNAL TABLE IF NOT EXISTS `${tableName}`
- |(${colsSql}) PARTITIONED BY (${partitionsSql})
- |ROW FORMAT DELIMITED
- |FIELDS TERMINATED BY ','
- |LINES TERMINATED BY '\n'
- |STORED AS TEXTFILE
- |LOCATION '${targetDumpDir}'""".stripMargin
- (sql, targetDumpDir)
- }
-
-
- def genPartitionHdfsPath(partitions: List[(String, Any)]): String = {
- partitions.map(prtn => s"${prtn._1}=${prtn._2}").mkString("/")
- }
-
- def genAddPartitionSql(tableName: String, partitions: List[(String, Any)]): String = {
- val partitionSql = partitions.map(prtn => (s"`${prtn._1}`=${prtn._2}")).mkString(", ")
- val sql = s"""ALTER TABLE `${tableName}` ADD IF NOT EXISTS PARTITION (${partitionSql})"""
- sql
- }
-
- def genDropPartitionSql(tableName: String, partitions: List[(String, Any)]): String = {
- val partitionSql = partitions.map(prtn => (s"`${prtn._1}`=${prtn._2}")).mkString(", ")
- val sql = s"""ALTER TABLE `${tableName}` DROP IF EXISTS PARTITION (${partitionSql})"""
- sql
- }
-
- def genSelectSql(tableName: String, partitionsRange: List[(String, Any)]): String = {
- val whereClause = partitionsRange match {
- case Nil => ""
- case _ => {
- val clause = partitionsRange.map { prtn =>
- prtn match {
- case (name, range: (Any, Any)) => s"""`${name}` BETWEEN '${range._1}' and '${range._2}'"""
- case (name, value) => s"""`${name}`='${value}'"""
- case _ => ""
- }
- }.filter(_.nonEmpty).mkString(" AND ")
- s"WHERE ${clause}"
- }
- }
- val sql = s"""SELECT * FROM `${tableName}` ${whereClause}"""
- sql
- }
-
- def sampleFunc[T](src: RDD[T], ratio: Double, count: Int = 0): RDD[T] = {
- if (src.isEmpty) {
- println("===== empty source =====")
- src
- } else {
- val sample = src.sample(false, ratio, System.nanoTime)
- if (sample.isEmpty) {
- if (count < 5) {
- sampleFunc(src, ratio, count + 1)
- } else {
- println("===== empty sample, get source =====")
- src
- }
- } else sample
- }
- }
-
-
-
- def getTimeMs(t: Int, unit: String): Long = {
- val lt = t.toLong
- unit match {
- case "ms" => lt
- case "sec" => lt * 1000
- case "min" => lt * 60 * 1000
- case "hour" => lt * 60 * 60 * 1000
- case "day" => lt * 24 * 60 * 60 * 1000
- case _ => lt * 60 * 1000
- }
- }
-
- def getTimeToUnit(lt: Long, unit: String): Long = {
- unit match {
- case "ms" => lt
- case "sec" => lt / 1000
- case "min" => lt / (60 * 1000)
- case "hour" => lt / (60 * 60 * 1000)
- case "day" => lt / (24 * 60 * 60 * 1000)
- case _ => lt / (60 * 1000)
- }
- }
-
- def updateDumpMin(min: Long): Unit = {
- this.dumpMin = min
- }
-
- def getDumpedMin(): Long = {
- this.dumpMin - 1
- }
-
- def dumpReady(beginMin: Long, dumpedMin: Long): Boolean = {
- beginMin <= dumpedMin
- }
-
-
- def toTuple[A <: AnyRef](as: Seq[A]): Product = {
- val tupleClass = Class.forName("scala.Tuple" + as.size)
- tupleClass.getConstructors.apply(0).newInstance(as: _*).asInstanceOf[Product]
- }
-
- def initNextFireTime(t: Long, nextInterval: Long, precInterval: Long): Unit = {
- val nt = nextInterval + t
- this.nextFireTime = nt / precInterval * precInterval
- }
-
- def updateNextFireTime(nextInterval: Long): Unit = {
- this.nextFireTime += nextInterval
- }
-
- def getNextFireTime(): Long = {
- this.nextFireTime
- }
-
- def getDumpValues(data: Map[String, Any], keys: List[String]): List[String] = {
- keys.map { k =>
- data.get(k) match {
- case Some(v) => if (v != null) v.toString else ""
- case _ => ""
- }
- }
- }
-
- def getStringFromSchema(datas: (Map[String, Any], Map[String, Any]), dumpConfigParam: DumpParam, dumpWrapper: DumpWrapper): List[Option[_]] = {
- val (dt, info) = datas
- val schema = dumpConfigParam.schema
- val schemaValues = schema.map { s =>
- dt.get(s.name)
- }
- val infoValues = dumpWrapper.wrapValues(info)
- schemaValues ::: infoValues
- }
-
- def getCheckMin(): Long = {
- this.checkMin
- }
-
- def updateCheckMin(min: Long): Unit = {
- this.checkMin = min
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/core/AccuracyCore.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/core/AccuracyCore.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/core/AccuracyCore.scala
deleted file mode 100644
index b7a7c8e..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/algo/core/AccuracyCore.scala
+++ /dev/null
@@ -1,92 +0,0 @@
-package org.apache.griffin.algo.core
-
-import org.apache.griffin.dump._
-import org.apache.griffin.record.result.AccuResult
-import org.apache.spark.rdd.RDD
-
-
-object AccuracyCore {
-
- type T = Map[String, Any]
-
- def accuracy(allkvs: RDD[(Product, (Iterable[(T, T)], Iterable[(T, T)]))]
- ): (AccuResult, RDD[(Product, (T, T))], RDD[(Product, (T, T))]) = {
- val result: RDD[(Long, Long, List[(Product, (T, T))], List[(Product, (T, T))])] = allkvs.map { kv =>
- val (k, (srcData, tgtData)) = kv
-// val srcCount = srcData.size
- val tgtCount = tgtData.size
- val tgt: Set[T] = tgtData.map(_._1).toSet // only get value
-
- // result: (missCount, matchCount, missDataList, matchDataList)
- val rslt = srcData.foldLeft((0L, 0L, List[(Product, (T, T))](), List[(Product, (T, T))]())) { (r, mapPair) =>
- val (valueMap, infoMap) = mapPair
- if (tgt.contains(valueMap)) {
- val newInfoMap = infoMap + (MismatchInfo.key -> "matched")
- val matchItem = (k, (valueMap, newInfoMap))
- (r._1, r._2 + 1, r._3, r._4 :+ matchItem)
- } else {
- val newInfoMap = if (tgtCount == 0) {
- infoMap + (MismatchInfo.key -> "no target")
- } else if (tgtCount == 1) {
- infoMap + (MismatchInfo.key -> ("target data: " + tgt.head.toString))
- } else {
- infoMap + (MismatchInfo.key -> "not match")
- }
- val missItem = (k, (valueMap, newInfoMap))
- (r._1 + 1, r._2, r._3 :+ missItem, r._4)
- }
- }
- rslt
- }
-
- val missRdd = result.flatMap(_._3)
- val matchRdd = result.flatMap(_._4)
-
- def seq(cnt: (Long, Long), rcd: (Long, Long, Any, Any)): (Long, Long) = {
- (cnt._1 + rcd._1, cnt._2 + rcd._2)
- }
- def comb(c1: (Long, Long), c2: (Long, Long)): (Long, Long) = {
- (c1._1 + c2._1, c1._2 + c2._2)
- }
- val countPair = result.aggregate((0L, 0L))(seq, comb)
-
- (AccuResult(countPair._1, (countPair._1 + countPair._2)), missRdd, matchRdd)
- }
-
-// def accuracy(allkvs: RDD[(Product, (Iterable[Map[String, Any]], Iterable[Map[String, Any]]))]): (AccuResult, List[String], RDD[(Product, Map[String, Any])]) = {
-// val result: RDD[((Long, Long), List[String], List[(Product, Map[String, Any])])] = allkvs.map { kv =>
-// val (k, (srcData, tgtData)) = kv
-// val srcCount = srcData.size
-// val tgtCount = tgtData.size
-// val tgt = tgtData.toSet
-// val rslt = srcData.foldLeft((0L, List[String](), List[(Product, Map[String, Any])]())) { (r, mp) =>
-// if (tgt.contains(mp)) {
-// r
-// } else {
-// val mp1 = if (tgtCount == 0) {
-// mp + ("_error_" -> "no target")
-// } else if (tgtCount == 1) {
-// mp + ("_error_" -> tgt.head.toString)
-// } else {
-// mp + ("_error_" -> "not match")
-// }
-// (r._1 + 1, mp1.toString :: r._2, (k, mp) :: r._3)
-// }
-// }
-// ((rslt._1, srcCount.toLong), rslt._2, rslt._3)
-// }
-//
-// val missingRdd = result.flatMap(_._3)
-//
-// def seq(cnt: (Long, Long, List[String]), rcd: ((Long, Long), List[String], Any)): (Long, Long, List[String]) = {
-// (cnt._1 + rcd._1._1, cnt._2 + rcd._1._2, cnt._3 ::: rcd._2)
-// }
-// def comb(c1: (Long, Long, List[String]), c2: (Long, Long, List[String])): (Long, Long, List[String]) = {
-// (c1._1 + c2._1, c1._2 + c2._2, c1._3 ::: c2._3)
-// }
-// val missingCount = result.aggregate((0L, 0L, List[String]()))(seq, comb)
-//
-// (AccuResult(missingCount._1, missingCount._2), missingCount._3, missingRdd)
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CacheProcesser.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CacheProcesser.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CacheProcesser.scala
deleted file mode 100644
index c70ce77..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CacheProcesser.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.apache.griffin.cache
-
-import org.apache.griffin.config.params.RetryParam
-
-import scala.collection.mutable.{Map => MutableMap}
-
-case class CacheProcesser(retryParam: RetryParam) {
-
- val cacheGroup: MutableMap[Long, CachedData] = MutableMap()
-
- val needRetry = retryParam.needRetry
- val nextRetryInterval = retryParam.nextRetryInterval * 1000
-
- def cache(timeStamp: Long, cacheData: CachedData): Unit = {
- if (!cacheData.cacheEnd) {
- cacheGroup.get(timeStamp) match {
- case Some(cd) => {
- println("unnecessary to cache")
- }
- case _ => {
- cacheGroup += (timeStamp -> cacheData)
- println(s"add cache ${timeStamp}: ${cacheData.result}")
- }
- }
- } else {
- println("unnecessary to cache")
- }
- }
-
-// def process(time: Long): Unit = {
-// val curCacheGroup = cacheGroup.toMap
-// val fireCaches = curCacheGroup.filter { pair => (time >= pair._2.nextFireTime) }
-// println(s"=== retry group count: ${fireCaches.size} ===")
-// fireCaches.foreach { pair =>
-// val (_, cacheData) = pair
-// println(s"=== retry: ${pair._1} ===")
-// cacheData.recalcAndCache(time)
-// cacheData.nextFireTime = time + nextRetryInterval
-// }
-// }
-
- def refresh(t: Long): Unit = {
- val curCacheGroup = cacheGroup.toMap
- curCacheGroup.foreach(_._2.curTime = t)
- val deadCache = curCacheGroup.filter(_._2.cacheEnd)
- println(s"=== dead cache group count: ${deadCache.size} ===")
- deadCache.keySet.foreach(cacheGroup -= _)
- }
-
- def getCache(timeStamp: Long): Option[CachedData] = {
- cacheGroup.get(timeStamp)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedAccuData.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedAccuData.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedAccuData.scala
deleted file mode 100644
index 0d939e5..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedAccuData.scala
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.apache.griffin.cache
-
-import org.apache.spark.rdd.RDD
-import org.apache.griffin.record.result.AccuResult
-
-case class CachedAccuData() extends CachedData {
-
- type T = AccuResult
-
-// var validMinRange: (Long, Long) = _
-
-// var proc: (Long) => (T, RDD[(Product, Map[String, Any])], (Long, Long)) => (T, RDD[(Product, Map[String, Any])], (Long, Long)) = _
-
-// def recalcAndCache(t: Long): Unit = {
-// val res = proc(t)(result, rdd, validMinRange)
-// unpersist
-// result = res._1
-// rdd = res._2
-// validMinRange = res._3
-// persist
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedData.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedData.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedData.scala
deleted file mode 100644
index 64474f8..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/cache/CachedData.scala
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.apache.griffin.cache
-
-import org.apache.spark.rdd.RDD
-import org.apache.griffin.record.result.Result
-
-
-trait CachedData {
-
- type T <: Result
-
- var result: T = _
-
- var curTime: Long = _
- var deadTime: Long = _
-// var rdd: RDD[(Product, Map[String, Any])] = _
-
-// var nextFireTime: Long = _
-
-// def persist(): Unit = { rdd.cache }
-// def unpersist(): Unit = { rdd.unpersist() }
-
-// def recalcAndCache(t: Long): Unit
- def cacheEnd(): Boolean = {
- (curTime > deadTime) || (result != null && !result.needCalc)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigFileReader.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigFileReader.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigFileReader.scala
deleted file mode 100644
index f7e6fdf..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigFileReader.scala
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.apache.griffin.config
-
-import org.apache.griffin.config.params._
-import org.apache.griffin.utils.JsonUtil
-
-case class ConfigFileReader(file: String) extends ConfigReader {
-
-// def readConfig[T <: ConfigParam](implicit m : Manifest[T]): T = {
-// val lines = scala.io.Source.fromFile(file).mkString
-// val param = JsonUtil.fromJson[T](lines)
-// param
-// }
-
- def readConfig: AllParam = {
- val lines = scala.io.Source.fromFile(file).mkString
- val param = JsonUtil.fromJson2AllParam(lines)
- param
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigReader.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigReader.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigReader.scala
deleted file mode 100644
index 5d87976..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/ConfigReader.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.config
-
-import org.apache.griffin.config.params.ConfigParam
-
-trait ConfigReader extends Serializable {
-
-// def readConfig[T <: ConfigParam](implicit m : Manifest[T]): T
-
- def readConfig: ConfigParam
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AccuracyParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AccuracyParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AccuracyParam.scala
deleted file mode 100644
index f4a263d..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AccuracyParam.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class AccuracyParam( @JsonProperty("mapping") mapping: List[MappingParam],
- @JsonProperty("match.valid.time") matchValidTime: TimeRangeParam
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AllParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AllParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AllParam.scala
deleted file mode 100644
index 7c7be6d..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/AllParam.scala
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class AllParam( @JsonProperty("spark") sparkParam: SparkParam,
- @JsonProperty("type.config") typeConfigParam: TypeConfigParam,
- @JsonProperty("dataAsset") dataAssetParamMap: Map[String, DataAssetParam],
- @JsonProperty("dq.config") dqConfigParam: DqConfigParam,
- @JsonProperty("recorder") recorderParam: RecorderParam,
- @JsonProperty("retry") retryParam: RetryParam
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ConfigParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ConfigParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ConfigParam.scala
deleted file mode 100644
index f97d071..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ConfigParam.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.griffin.config.params
-
-
-trait ConfigParam extends Serializable {
-
- def merge(cp: ConfigParam): Unit = {}
-
- def validate(): Boolean = true
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DataAssetParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DataAssetParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DataAssetParam.scala
deleted file mode 100644
index 3647d8d..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DataAssetParam.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class DataAssetParam( @JsonProperty("kafka.config") kafkaConfig: Map[String, String],
- @JsonProperty("topics") topics: String,
- @JsonProperty("pre.process") prepParam: PrepParam,
- @JsonProperty("dump.config") dumpConfigParam: DumpParam,
- @JsonProperty("schema") schema: List[SchemaFieldParam]
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqConfigParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqConfigParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqConfigParam.scala
deleted file mode 100644
index 789bbb8..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqConfigParam.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class DqConfigParam( @JsonProperty("accuracy") accuracy: AccuracyParam
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqType.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqType.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqType.scala
deleted file mode 100644
index c524614..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DqType.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.config.params
-
-sealed class DqType(val tp: String)
-
-case object AccuracyType extends DqType("accuracy")
-case object ProfileType extends DqType("profile")
-case object ValidityType extends DqType("validity")
-
-object DqType {
- def parse(tp: String): DqType = new DqType(tp)
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DumpParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DumpParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DumpParam.scala
deleted file mode 100644
index bea1ea6..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/DumpParam.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class DumpParam( @JsonProperty("dir") dir: String,
-// @JsonProperty("field.separator") fieldSep: String,
-// @JsonProperty("line.separator") lineSep: String,
- @JsonProperty("table.name") tableName: String,
- @JsonProperty("schema") schema: List[SchemaFieldParam]
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/MappingParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/MappingParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/MappingParam.scala
deleted file mode 100644
index 80f94f4..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/MappingParam.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class MappingParam( @JsonProperty("source.name") sourceName: String,
- @JsonProperty("target.name") targetName: String,
- @JsonProperty("isPK") isPK: Boolean
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseConfigParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseConfigParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseConfigParam.scala
deleted file mode 100644
index bf21d20..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseConfigParam.scala
+++ /dev/null
@@ -1,10 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class ParseConfigParam( @JsonProperty("out.schema") schema: List[SchemaFieldParam]
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseParam.scala
deleted file mode 100644
index d7d3d22..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/ParseParam.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class ParseParam( @JsonProperty("in.type") inType: String,
- @JsonProperty("out.type") outType: String,
- @JsonProperty("config") configParam: ParseConfigParam
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/PrepParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/PrepParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/PrepParam.scala
deleted file mode 100644
index d8a186d..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/PrepParam.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class PrepParam( @JsonProperty("sample") sampleParam: SampleParam,
- @JsonProperty("parse") parseParam: ParseParam
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RecorderParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RecorderParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RecorderParam.scala
deleted file mode 100644
index e407a21..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RecorderParam.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class RecorderParam( @JsonProperty("types") types: List[String],
- @JsonProperty("metric.name") metricName: String,
- @JsonProperty("config") config: Map[String, String]
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RetryParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RetryParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RetryParam.scala
deleted file mode 100644
index 46ebe07..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/RetryParam.scala
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class RetryParam( @JsonProperty("need.retry") needRetry: Boolean,
- @JsonProperty("next.retry.seconds") nextRetryInterval: Long,
- @JsonProperty("interval.seconds") interval: Long
- ) extends ConfigParam {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a5d54703/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/SampleParam.scala
----------------------------------------------------------------------
diff --git a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/SampleParam.scala b/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/SampleParam.scala
deleted file mode 100644
index 3749702..0000000
--- a/griffin-streaming-model/src/main/scala/org/apache/griffin/config/params/SampleParam.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.apache.griffin.config.params
-
-import com.fasterxml.jackson.annotation.{JsonInclude, JsonProperty}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-
-@JsonInclude(Include.NON_NULL)
-case class SampleParam( @JsonProperty("need.sample") needSample: Boolean,
- @JsonProperty("ratio") ratio: Double
- ) extends ConfigParam {
-
-}