You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/11 04:05:09 UTC
[26/29] DRILL-221,
part 3. More license updates. Automatically run Rat plugin with
appropriate excludes. Move Synth-log to sandbox until it is part of main mvn
build to avoid weird RAT problems.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/contrib/synth-log/src/test/java/org/apache/drill/synth/TermGeneratorTest.java
----------------------------------------------------------------------
diff --git a/contrib/synth-log/src/test/java/org/apache/drill/synth/TermGeneratorTest.java b/contrib/synth-log/src/test/java/org/apache/drill/synth/TermGeneratorTest.java
deleted file mode 100644
index b983eb2..0000000
--- a/contrib/synth-log/src/test/java/org/apache/drill/synth/TermGeneratorTest.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.synth;
-
-import com.google.common.base.Function;
-import com.google.common.collect.*;
-import org.apache.commons.math3.distribution.NormalDistribution;
-import org.apache.mahout.math.stats.LogLikelihood;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TermGeneratorTest {
-
- private static final WordGenerator WORDS = new WordGenerator("word-frequency-seed", "other-words");
-
- @Test
- public void generateTerms() {
- TermGenerator x = new TermGenerator(WORDS, 1, 0.8);
- final Multiset<String> counts = HashMultiset.create();
- for (int i = 0; i < 10000; i++) {
- counts.add(x.sample());
- }
-
- assertEquals(10000, counts.size());
- assertTrue("Should have some common words", counts.elementSet().size() < 10000);
- List<Integer> k = Lists.newArrayList(Iterables.transform(counts.elementSet(), new Function<String, Integer>() {
- public Integer apply(String s) {
- return counts.count(s);
- }
- }));
-// System.out.printf("%s\n", Ordering.natural().reverse().sortedCopy(k).subList(0, 30));
-// System.out.printf("%s\n", Iterables.transform(Iterables.filter(counts.elementSet(), new Predicate<String>() {
-// public boolean apply(String s) {
-// return counts.count(s) > 100;
-// }
-// }), new Function<String, String>() {
-// public String apply(String s) {
-// return s + ":" + counts.count(s);
-// }
-// }));
- assertEquals(1, Ordering.natural().leastOf(k, 1).get(0).intValue());
- assertTrue(Ordering.natural().greatestOf(k, 1).get(0) > 300);
- assertTrue(counts.count("the") > 300);
- }
-
- @Test
- public void distinctVocabularies() {
- TermGenerator x1 = new TermGenerator(WORDS, 1, 0.8);
- final Multiset<String> k1 = HashMultiset.create();
- for (int i = 0; i < 50000; i++) {
- k1.add(x1.sample());
- }
-
- TermGenerator x2 = new TermGenerator(WORDS, 1, 0.8);
- final Multiset<String> k2 = HashMultiset.create();
- for (int i = 0; i < 50000; i++) {
- k2.add(x2.sample());
- }
-
- final NormalDistribution normal = new NormalDistribution();
- List<Double> scores = Ordering.natural().sortedCopy(Iterables.transform(k1.elementSet(),
- new Function<String, Double>() {
- public Double apply(String s) {
- return normal.cumulativeProbability(LogLikelihood.rootLogLikelihoodRatio(k1.count(s), 50000 - k1.count(s), k2.count(s), 50000 - k2.count(s)));
- }
- }));
- int n = scores.size();
-// System.out.printf("%.5f, %.5f, %.5f, %.5f, %.5f, %.5f, %.5f", scores.get(0), scores.get((int) (0.05*n)), scores.get(n / 4), scores.get(n / 2), scores.get(3 * n / 4), scores.get((int) (0.95 * n)), scores.get(n - 1));
- int i = 0;
- for (Double score : scores) {
- if (i % 10 == 0) {
- System.out.printf("%.6f\t%.6f\n", (double) i / n, score);
- }
-
- i++;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/contrib/synth-log/src/test/java/org/apache/drill/synth/WordGeneratorTest.java
----------------------------------------------------------------------
diff --git a/contrib/synth-log/src/test/java/org/apache/drill/synth/WordGeneratorTest.java b/contrib/synth-log/src/test/java/org/apache/drill/synth/WordGeneratorTest.java
deleted file mode 100644
index 0bfdf06..0000000
--- a/contrib/synth-log/src/test/java/org/apache/drill/synth/WordGeneratorTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.synth;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class WordGeneratorTest {
- @Test
- public void checkRealWords() {
- WordGenerator words = new WordGenerator("word-frequency-seed", "other-words");
- for (int i = 0; i < 20000; i++) {
- assertFalse(words.getString(i).matches("-[0-9]+"));
- }
-
- for (int i = 0; i < 1000; i++) {
- String w = words.getString(i + 200000);
- assertTrue(w.matches(".*-[0-9]+"));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index b855e90..972e202 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.0.0-m1-SNAPSHOT</version>
+ <version>1.0.0-m1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill</groupId>
@@ -72,6 +72,20 @@
<build>
<plugins>
<plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <inherited>true</inherited>
+ <configuration>
+ <excludes>
+ <exclude>**/conffiles</exclude>
+ <exclude>**/control</exclude>
+ <exclude>**/*.json</exclude>
+ <exclude>**/hadoop-excludes.txt</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/distribution/src/assemble/bin.xml
----------------------------------------------------------------------
diff --git a/distribution/src/assemble/bin.xml b/distribution/src/assemble/bin.xml
index fe76c67..45fd1cc 100644
--- a/distribution/src/assemble/bin.xml
+++ b/distribution/src/assemble/bin.xml
@@ -1,3 +1,20 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/distribution/src/deb/control/conffiles
----------------------------------------------------------------------
diff --git a/distribution/src/deb/control/conffiles b/distribution/src/deb/control/conffiles
index a9d1fda..1a297aa 100644
--- a/distribution/src/deb/control/conffiles
+++ b/distribution/src/deb/control/conffiles
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
/etc/drill/conf/drill-override.conf
/etc/drill/conf/logback.xml
-/etc/drill/conf/drill-env.sh
\ No newline at end of file
+/etc/drill/conf/drill-env.sh
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/distribution/src/resources/logback.xml
----------------------------------------------------------------------
diff --git a/distribution/src/resources/logback.xml b/distribution/src/resources/logback.xml
index feb75b3..3ee676b 100644
--- a/distribution/src/resources/logback.xml
+++ b/distribution/src/resources/logback.xml
@@ -1,4 +1,14 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
<configuration>
<appender name="SOCKET"
class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/bufferl/pom.xml
----------------------------------------------------------------------
diff --git a/exec/bufferl/pom.xml b/exec/bufferl/pom.xml
index d52cae9..610a723 100644
--- a/exec/bufferl/pom.xml
+++ b/exec/bufferl/pom.xml
@@ -16,7 +16,7 @@
<parent>
<groupId>org.apache.drill.exec</groupId>
<artifactId>exec-parent</artifactId>
- <version>1.0.0-m1-SNAPSHOT</version>
+ <version>1.0.0-m1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.exec</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 1c5af4b..d912ca9 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>exec-parent</artifactId>
<groupId>org.apache.drill.exec</groupId>
- <version>1.0.0-m1-SNAPSHOT</version>
+ <version>1.0.0-m1-incubating-SNAPSHOT</version>
</parent>
<artifactId>java-exec</artifactId>
<name>exec/Java Execution Engine</name>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp
index ea27863..cd2b2cc 100644
--- a/exec/java-exec/src/main/codegen/config.fmpp
+++ b/exec/java-exec/src/main/codegen/config.fmpp
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
data: {
vv: tdd(../data/ValueVectorTypes.tdd),
compare: tdd(../data/CompareTypes.tdd),
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index c6ee258..760b0f9 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
{
types: [
{from: "Int", to: "BigInt"},
@@ -14,4 +30,4 @@
{to: "BigInt", from: "Float8" , explicit: "long"}
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/CompareTypes.tdd b/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
index 9c26a3f..f384d52 100644
--- a/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
+++ b/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
@@ -1,6 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
{
types: [
{comparables: ["Float4", "Float8", "Int", "BigInt", "NullableFloat4", "NullableFloat8", "NullableInt", "NullableBigInt"], mode: "fixed"},
{comparables: ["VarChar", "VarBinary", "NullableVarChar", "NullableVarBinary"], mode: "var"}
]
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
index d201671..88a5974 100644
--- a/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
@@ -1,3 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
{
modes: [
{name: "Optional", prefix: "Nullable"},
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 2029280..3e5a156 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -1,3 +1,14 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/logback.xml b/exec/java-exec/src/main/resources/logback.xml
index feb75b3..35f0260 100644
--- a/exec/java-exec/src/main/resources/logback.xml
+++ b/exec/java-exec/src/main/resources/logback.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<appender name="SOCKET"
class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/main/sh/runbit
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/sh/runbit b/exec/java-exec/src/main/sh/runbit
index 4e1b599..4c4aa21 100755
--- a/exec/java-exec/src/main/sh/runbit
+++ b/exec/java-exec/src/main/sh/runbit
@@ -1,5 +1,19 @@
#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
if [ -z $JAVA_HOME ]
then
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/logback.xml b/exec/java-exec/src/test/resources/logback.xml
index 206a469..dd32405 100644
--- a/exec/java-exec/src/test/resources/logback.xml
+++ b/exec/java-exec/src/test/resources/logback.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<appender name="SOCKET"
class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
@@ -25,4 +41,4 @@
<appender-ref ref="STDOUT" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/test/sh/logback.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/sh/logback.xml b/exec/java-exec/src/test/sh/logback.xml
index 6f2928a..ec7ea44 100644
--- a/exec/java-exec/src/test/sh/logback.xml
+++ b/exec/java-exec/src/test/sh/logback.xml
@@ -1,4 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
<configuration>
<appender name="SOCKET"
class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/java-exec/src/test/sh/runbit
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/sh/runbit b/exec/java-exec/src/test/sh/runbit
index 8a54006..7e4e73b 100755
--- a/exec/java-exec/src/test/sh/runbit
+++ b/exec/java-exec/src/test/sh/runbit
@@ -1,5 +1,20 @@
#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
PROJECT_ROOT=../../../
mvn dependency:build-classpath -f=$PROJECT_ROOT/pom.xml -Dmdep.outputFile=target/sh/cp.txt
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/pom.xml b/exec/pom.xml
index f0601dc..3859fb0 100644
--- a/exec/pom.xml
+++ b/exec/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>drill-root</artifactId>
<groupId>org.apache.drill</groupId>
- <version>1.0.0-m1-SNAPSHOT</version>
+ <version>1.0.0-m1-incubating-SNAPSHOT</version>
</parent>
<groupId>org.apache.drill.exec</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/ref/pom.xml
----------------------------------------------------------------------
diff --git a/exec/ref/pom.xml b/exec/ref/pom.xml
index c7f74e0..1a4c809 100644
--- a/exec/ref/pom.xml
+++ b/exec/ref/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>exec-parent</artifactId>
<groupId>org.apache.drill.exec</groupId>
- <version>1.0.0-m1-SNAPSHOT</version>
+ <version>1.0.0-m1-incubating-SNAPSHOT</version>
</parent>
<artifactId>ref</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/ref/src/main/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/exec/ref/src/main/resources/logback-test.xml b/exec/ref/src/main/resources/logback-test.xml
index 9a53c12..3d970a1 100644
--- a/exec/ref/src/main/resources/logback-test.xml
+++ b/exec/ref/src/main/resources/logback-test.xml
@@ -1,4 +1,14 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
<configuration>
<appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
@@ -30,4 +40,4 @@
</root>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/exec/ref/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/exec/ref/src/test/resources/logback.xml b/exec/ref/src/test/resources/logback.xml
index 10547b2..39bb0c8 100644
--- a/exec/ref/src/test/resources/logback.xml
+++ b/exec/ref/src/test/resources/logback.xml
@@ -1,4 +1,14 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
<configuration>
<appender name="SOCKET" class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
@@ -43,4 +53,4 @@
<appender-ref ref="FILE" />
</root>
-</configuration>
\ No newline at end of file
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30bffd0..e055bbe 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,20 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -27,7 +21,7 @@
<groupId>org.apache.drill</groupId>
<artifactId>drill-root</artifactId>
- <version>1.0.0-m1-SNAPSHOT</version>
+ <version>1.0.0-m1-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Drill Root POM</name>
@@ -96,6 +90,34 @@
<build>
<plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>rat-checks</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+
+ <excludeSubprojects>false</excludeSubprojects>
+ <excludes>
+ <exclude>**/*.md</exclude>
+ <exclude>sandbox/**</exclude>
+ <exclude>**/*.json</exclude>
+ <exclude>**/drill-*.conf</exclude>
+ <exclude>**/.buildpath</exclude>
+ <exclude>**/*.proto</exclude>
+ <exclude>**/*.fmpp</exclude>
+ <exclude>**/*.tdd</exclude>
+
+ </excludes>
+ </configuration>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -183,6 +205,11 @@
<plugins>
<plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>0.10</version>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>2.6</version>
@@ -215,15 +242,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>0.10</version>
- <configuration>
- <property>rat.excludeSubprojects</property>
- </configuration>
- </plugin>
-
<!--This plugin's configuration is used to store Eclipse m2e settings
only. It has no influence on the Maven build itself. -->
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/README.md
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/README.md b/sandbox/synth-log/README.md
new file mode 100644
index 0000000..b4efc63
--- /dev/null
+++ b/sandbox/synth-log/README.md
@@ -0,0 +1,55 @@
+log-synth
+=========
+
+The basic idea here is to have a random log generator build fairly realistic log files for analysis. The analyses specified here are fairly typical use cases for trying to figure out where the load on a web-site is coming from.
+
+How to Run It
+============
+
+Install Java 7, maven and get this software using git.
+
+On a mac, this can help get the right version of Java
+
+ export JAVA_HOME=$(/usr/libexec/java_home)
+
+Then do this to build a jar file with all dependencies included
+
+ mvn package
+
+Then use this to write one million log lines into the file "log" and to write the associated user database into the file "users".
+
+ java -cp target/log-synth-0.1-SNAPSHOT-jar-with-dependencies.jar org.apache.drill.synth.Main 1M log users
+
+This program will produce a line of output on the standard output for each 10,000 lines of log produced. Each line will contain the number of log lines produced so far and the number of unique users in the user profile database.
+
+
+The Data Source
+==============
+The data source here is a set of heavily biased random numbers to generate traffic sources, response times and queries. In order to give a realistic long-tail experience the data are generated using special random number generators available in the Mahout library.
+
+There are three basic entities involved in the random process that generates these logs that are IP addresses, users and queries. Users have a basic traffic rate and a variable number of users sit behind each IP address. Queries are composed of words which are generated somewhat differently by each user. The response time for each query is determined based on the terms in the queries with a very few terms causing much longer queries than others. Each log line contains an IP address, a user cookie, a query and a response time.
+
+Logs of various sizes can be generated using the generator tools.
+
+The Queries
+==============
+The general goal of the queries is to find out what and/or who is causing long query times and where lots of traffic is coming from.
+
+The questions we would like to answer include:
+
+* What are the top IP addresses by request count?
+* What are the top IP addresses by unique user?
+* What are the most common search terms?
+* What are the most common search terms in the slowest 5% of the queries?
+* What is the daily number of searches, (approximate) number of unique users, (approximate) number of unique IP addresses and distribution of response times (average, min, max, 25, 50 and 75%-iles).
+
+Methods
+========
+The general process for generating log lines is to select a user, possibly one we have not seen before. If the user is new, then we need to select an IP address for the user. Otherwise, we remember the IP address for each user.
+
+Queries have an overall frequency distribution that is long-tailed, but each user has a variation on that distribution. In order to model this, we sample each user's queries from a per-user Pittman-Yor process. In order to make users have similar query term distributions, each user's query term distribution is initialized from a Pittman-Yor process that has already been sampled a number of times.
+
+We also need to maintain an average response time per term. The response time for each query is exponentially distributed with a mean equal to the sum of the average response times for the terms. Response times for words are sampled either from an exponential distribution, from a log-gamma distribution or from a gamma distribution with a moderately low shape parameter so that we can have interestingly long tails for response time.
+
+Users are assigned to IP addresses using a Pittman-Yor process with a discount of 0.9. This gives long-tailed distribution to the number of users per IP address. This results in 90% of all IP addresses having only a single user.
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/pom.xml b/sandbox/synth-log/pom.xml
new file mode 100644
index 0000000..b46d869
--- /dev/null
+++ b/sandbox/synth-log/pom.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>log-synth</groupId>
+ <artifactId>log-synth</artifactId>
+ <version>0.1-SNAPSHOT</version>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-math</artifactId>
+ <version>0.8-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.6.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>1.6.6</version>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.0</version>
+ <configuration>
+ <verbose>true</verbose>
+ <compilerVersion>1.7</compilerVersion>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java
new file mode 100644
index 0000000..54b671b
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import com.google.common.base.Preconditions;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.math.list.DoubleArrayList;
+import org.apache.mahout.math.random.Sampler;
+
+import java.util.Random;
+
+/**
+ *
+ * Generates samples from a generalized Chinese restaurant process (or Pittman-Yor process).
+ *
+ * The number of values drawn exactly once will asymptotically be equal to the discount parameter
+ * as the total number of draws T increases without bound. The number of unique values sampled will
+ * increase as O(alpha * log T) if discount = 0 or O(alpha * T^discount) for discount > 0.
+ */
+public final class ChineseRestaurant implements Sampler<Integer> {
+ private final double alpha;
+ private double weight = 0;
+ private double discount = 0;
+ private final DoubleArrayList weights = new DoubleArrayList();
+ private final Random rand = RandomUtils.getRandom();
+
+ /**
+ * Constructs a Dirichlet process sampler. This is done by setting discount = 0.
+ * @param alpha The strength parameter for the Dirichlet process.
+ */
+ public ChineseRestaurant(double alpha) {
+ this(alpha, 0);
+ }
+
+ /**
+ * Constructs a Pitman-Yor sampler.
+ *
+ * @param alpha The strength parameter that drives the number of unique values as a function of draws.
+ * @param discount The discount parameter that drives the percentage of values that occur once in a large sample.
+ */
+ public ChineseRestaurant(double alpha, double discount) {
+ Preconditions.checkArgument(alpha > 0);
+ Preconditions.checkArgument(discount >= 0 && discount <= 1);
+ this.alpha = alpha;
+ this.discount = discount;
+ }
+
+ public Integer sample() {
+ double u = rand.nextDouble() * (alpha + weight);
+ for (int j = 0; j < weights.size(); j++) {
+ // select existing options with probability (w_j - d) / (alpha + w)
+ if (u < weights.get(j) - discount) {
+ weights.set(j, weights.get(j) + 1);
+ weight++;
+ return j;
+ } else {
+ u -= weights.get(j) - discount;
+ }
+ }
+
+ // if no existing item selected, pick new item with probability (alpha - d*t) / (alpha + w)
+ // where t is number of pre-existing cases
+ weights.add(1);
+ weight++;
+ return weights.size() - 1;
+ }
+
+ /**
+ * @return the number of unique values that have been returned.
+ */
+ public int size() {
+ return weights.size();
+ }
+
+ /**
+ * @return the number draws so far.
+ */
+ public int count() {
+ return (int) weight;
+ }
+
+ /**
+ * @param j Which value to test.
+ * @return The number of times that j has been returned so far.
+ */
+ public int count(int j) {
+ Preconditions.checkArgument(j >= 0);
+
+ if (j < weights.size()) {
+ return (int) weights.get(j);
+ } else {
+ return 0;
+ }
+ }
+
+ public void setCount(int term, double count) {
+ while (weights.size() <= term) {
+ weights.add(0);
+ }
+ weight += (count - weights.get(term));
+ weights.set(term, count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogGenerator.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogGenerator.java
new file mode 100644
index 0000000..a030501
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogGenerator.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import org.apache.mahout.math.random.Sampler;
+
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Random;
+
+/**
+ * Generates kind of realistic log lines consisting of a user id (a cookie), an IP address and a query.
+ */
+public class LogGenerator implements Sampler<LogLine> {
+ private LongTail<InetAddress> ipGenerator = new LongTail<InetAddress>(1, 0.5) {
+ Random gen = new Random();
+
+ @Override
+ protected InetAddress createThing() {
+ int address = gen.nextInt();
+ try {
+ return Inet4Address.getByAddress(new byte[]{
+ (byte) (address >>> 24),
+ (byte) (0xff & (address >>> 16)),
+ (byte) (0xff & (address >>> 8)),
+ (byte) (0xff & (address))
+ });
+ } catch (UnknownHostException e) {
+ throw new RuntimeException("Can't happen with numeric IP address", e);
+ }
+ }
+ };
+
+ private WordGenerator words = new WordGenerator("word-frequency-seed", "other-words");
+ private TermGenerator terms = new TermGenerator(words, 1, 0.8);
+ private TermGenerator geo = new TermGenerator(new WordGenerator(null, "geo-codes"), 10, 0
+ );
+
+ private LongTail<User> userGenerator = new LongTail<User>(50000, 0) {
+ @Override
+ protected User createThing() {
+ return new User(ipGenerator.sample(), geo, terms);
+ }
+ };
+
+ public Iterable<User> getUsers() {
+ return userGenerator.getThings();
+ }
+
+ public LogLine sample() {
+ // pick a user
+ return new LogLine(userGenerator.sample());
+ }
+
+ public int getUserCount() {
+ return userGenerator.getThings().size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogLine.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogLine.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogLine.java
new file mode 100644
index 0000000..d15b53d
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/LogLine.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import java.net.InetAddress;
+import java.util.Formatter;
+import java.util.List;
+
+/**
+ * A log line contains a user id, an IP address and a query.
+ */
+public class LogLine {
+ private InetAddress ip;
+ private long cookie;
+ private List<String> query;
+
+ public LogLine(InetAddress ip, long cookie, List<String> query) {
+ this.cookie = cookie;
+ this.ip = ip;
+ this.query = query;
+ }
+
+ public LogLine(User user) {
+ this(user.getAddress(), user.getCookie(), user.getQuery());
+ }
+
+ @Override
+ public String toString() {
+ Formatter r = new Formatter();
+ r.format("{cookie:\"%08x\", ip:\"%s\", query:", cookie, ip.getHostAddress());
+ String sep = "[";
+ for (String term : query) {
+ r.format("%s\"%s\"", sep, term);
+ sep = ", ";
+ }
+ r.format("]}");
+ return r.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/LongTail.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
new file mode 100644
index 0000000..bf88852
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import com.google.common.collect.Lists;
+import org.apache.mahout.math.random.Sampler;
+
+import java.util.List;
+
+/**
+ * Samples from a set of things based on a long-tailed distribution. This converts
+ * the ChineseRestaurant distribution from a distribution over integers into a distribution
+ * over more plausible looking things like words.
+ */
+public abstract class LongTail<T> implements Sampler<T> {
+ private ChineseRestaurant base;
+ private List<T> things = Lists.newArrayList();
+
+ protected LongTail(double alpha, double discount) {
+ base = new ChineseRestaurant(alpha, discount);
+ }
+
+ public T sample() {
+ int n = base.sample();
+ while (n >= things.size()) {
+ things.add(createThing());
+ }
+ return things.get(n);
+ }
+
+ public ChineseRestaurant getBaseDistribution() {
+ return base;
+ }
+
+ protected abstract T createThing();
+
+ public List<T> getThings() {
+ return things;
+ }
+
+ public void setThing(int i, T thing) {
+ while (things.size() <= i) {
+ things.add(null);
+ }
+ things.set(i, thing);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/Main.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/Main.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/Main.java
new file mode 100644
index 0000000..e95b9a7
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/Main.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Create a query log with a specified number of log lines and an associated user profile database.
+ * <p/>
+ * Command line args include number of log lines to generate, the name of the log file to generate and the
+ * name of the file to store the user profile database in.
+ * <p/>
+ * Log lines and user profile entries are single line JSON.
+ */
+public class Main {
+ public static void main(String[] args) throws IOException {
+
+ int n = Integer.parseInt(args[0].replaceAll("[KMG]?$", ""));
+
+ switch (args[0].charAt(args[0].length() - 1)) {
+ case 'G':
+ n *= 1e9;
+ break;
+ case 'M':
+ n *= 1e6;
+ break;
+ case 'K':
+ n *= 1e3;
+ break;
+ default:
+ // no suffix leads here
+ break;
+ }
+
+ LogGenerator lg = new LogGenerator();
+ BufferedWriter log = Files.newWriter(new File(args[1]), Charsets.UTF_8);
+ for (int i = 0; i < n; i++) {
+ if (i % 10000 == 0) {
+ System.out.printf("%d %d\n", i, lg.getUserCount());
+ }
+ log.write(lg.sample().toString());
+ log.newLine();
+ }
+ log.close();
+
+ BufferedWriter profile = Files.newWriter(new File(args[2]), Charsets.UTF_8);
+ for (User user : lg.getUsers()) {
+ profile.write(user.toString());
+ profile.newLine();
+ }
+ profile.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/TermGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/TermGenerator.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/TermGenerator.java
new file mode 100644
index 0000000..9ddd61b
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/TermGenerator.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import org.apache.mahout.math.random.Sampler;
+
+/**
+ * Generate words at random from a specialized vocabulary. Every term generator's
+ * frequency distribution has a common basis, but each will diverge after initialization.
+ */
+public class TermGenerator implements Sampler<String> {
+ // the word generator handles the problem of making up new words
+ // it also provides the seed frequencies
+ private WordGenerator words;
+
+ private LongTail<String> distribution;
+
+ public TermGenerator(WordGenerator words, final int alpha, final double discount) {
+ this.words = words;
+ distribution = new LongTail<String>(alpha, discount) {
+ private int count = TermGenerator.this.words.size();
+
+ @Override
+ protected String createThing() {
+ return TermGenerator.this.words.getString(count++);
+ }
+ };
+
+ int i = 0;
+ for (String word : this.words.getBaseWeights().keySet()) {
+ distribution.getBaseDistribution().setCount(i, this.words.getBaseWeights().get(word));
+ distribution.setThing(i, word);
+ i++;
+ }
+
+ }
+
+ public String sample() {
+ return distribution.sample();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/User.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/User.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/User.java
new file mode 100644
index 0000000..194a666
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/User.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import com.google.common.collect.Lists;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.math.jet.random.Exponential;
+
+import java.net.InetAddress;
+import java.util.List;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: tdunning
+ * Date: 2/2/13
+ * Time: 6:15 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class User {
+ private Exponential queryLengthDistribution = new Exponential(0.4, RandomUtils.getRandom());
+
+ private long cookie = RandomUtils.getRandom().nextLong();
+
+ private TermGenerator terms;
+ private InetAddress address;
+ private String geoCode;
+
+ public User(InetAddress address, TermGenerator geoCoder, TermGenerator terms) {
+ this.terms = terms;
+ geoCode = geoCoder.sample();
+ this.address = address;
+ }
+
+ public InetAddress getAddress() {
+ return address;
+ }
+
+ public long getCookie() {
+ return cookie;
+ }
+
+ public List<String> getQuery() {
+ int n = queryLengthDistribution.nextInt() + 1;
+ List<String> r = Lists.newArrayList();
+ for (int i = 0; i < n; i++) {
+ r.add(terms.sample());
+ }
+ return r;
+ }
+
+ public String getGeoCode() {
+ return geoCode;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("{ip:\"%s\", cookie:\"%08x\", geo:\"%s\"}", address, cookie, geoCode);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/java/org/apache/drill/synth/WordGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/java/org/apache/drill/synth/WordGenerator.java b/sandbox/synth-log/src/main/java/org/apache/drill/synth/WordGenerator.java
new file mode 100644
index 0000000..0167308
--- /dev/null
+++ b/sandbox/synth-log/src/main/java/org/apache/drill/synth/WordGenerator.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.synth;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.io.LineProcessor;
+import com.google.common.io.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Emulates an infinite list of words, a prefix of which are taken from lists of plausible words. The first words
+ * are taken from a resource that has frequencies in it. These frequencies can be used to initialize term
+ * generators to a common language. The next batch of words are taken from a long list of words with no frequencies.
+ * After that, words are coined by using an integer count.
+ */
+public class WordGenerator {
+ private final Logger log = LoggerFactory.getLogger(WordGenerator.class);
+
+ private BufferedReader wordReader;
+ private final List<String> words = Lists.newArrayList();
+ private final Map<String, Integer> baseWeights = Maps.newLinkedHashMap();
+
+ public WordGenerator(String seed, String others) {
+ // read the common words
+ if (seed != null) {
+ try {
+ Resources.readLines(Resources.getResource(seed), Charsets.UTF_8,
+ new LineProcessor<Object>() {
+ private boolean header = true;
+ private final Splitter onTabs = Splitter.on("\t");
+
+ public boolean processLine(String s) throws IOException {
+ if (!s.startsWith("#")) {
+ if (!header) {
+ Iterator<String> fields = onTabs.split(s).iterator();
+ fields.next();
+ String word = fields.next();
+ words.add(word);
+ int count = (int) Math.rint(Double.parseDouble(fields.next()));
+ baseWeights.put(word, count);
+ } else {
+ header = false;
+ }
+ }
+ return true;
+ }
+
+ public Object getResult() {
+ return null;
+ }
+ });
+ } catch (IOException e) {
+ log.error("Can't read resource \"{}\", will continue without realistic words", seed);
+ }
+ }
+
+ try {
+ wordReader = new BufferedReader(Resources.newReaderSupplier(Resources.getResource(others), Charsets.UTF_8).getInput());
+ } catch (IOException e) {
+ log.error("Can't read resource \"{}\", will continue without realistic words", others);
+ wordReader = null;
+ }
+
+ }
+
+ public String getString(int n) {
+ if (n >= words.size()) {
+ synchronized (this) {
+ while (n >= words.size()) {
+ try {
+ String w = wordReader.readLine();
+ if (w != null) {
+ words.add(w);
+ } else {
+ words.add("w-" + n);
+ }
+ } catch (IOException e) {
+ log.error("Error reading other words resource", e);
+ words.add("w-" + n);
+ }
+ }
+ }
+ }
+ return words.get(n);
+ }
+
+ public Map<String, Integer> getBaseWeights() {
+ return baseWeights;
+ }
+
+ public int size() {
+ return words.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/resources/geo-codes
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/resources/geo-codes b/sandbox/synth-log/src/main/resources/geo-codes
new file mode 100644
index 0000000..f08d1fc
--- /dev/null
+++ b/sandbox/synth-log/src/main/resources/geo-codes
@@ -0,0 +1,297 @@
+AL
+AK
+AZ
+AR
+CA
+CO
+CT
+DE
+FL
+GA
+HI
+ID
+IL
+IN
+IA
+KS
+KY
+LA
+ME
+MD
+MA
+MI
+MN
+MS
+MO
+MT
+NE
+NV
+NH
+NJ
+NM
+NY
+NC
+ND
+OH
+OK
+OR
+PA
+RI
+SC
+SD
+TN
+TX
+UT
+VT
+VA
+WA
+WV
+WI
+WY
+AS
+DC
+FM
+GU
+MH
+MP
+PW
+PR
+VI
+AF
+AL
+DZ
+AS
+AD
+AO
+AI
+AQ
+AG
+AR
+AM
+AW
+AU
+AT
+AZ
+BS
+BH
+BD
+BB
+BY
+BE
+BZ
+BJ
+BM
+BT
+BO
+BA
+BW
+BV
+BR
+IO
+BN
+BG
+BF
+BI
+KH
+CM
+CA
+CV
+KY
+CF
+TD
+CL
+CN
+CX
+CC
+CO
+KM
+CG
+CD
+CK
+CR
+CI
+HR
+CU
+CY
+CZ
+DK
+DJ
+DM
+DO
+TP
+EC
+EG
+SV
+GQ
+ER
+EE
+ET
+FK
+FO
+FJ
+FI
+FR
+FX
+GF
+PF
+TF
+GA
+GM
+GE
+DE
+GH
+GI
+GR
+GL
+GD
+GP
+GU
+GT
+GN
+GW
+GY
+HT
+HM
+VA
+HN
+HK
+HU
+IS
+IN
+ID
+IR
+IQ
+IE
+IL
+IT
+JM
+JP
+JO
+KZ
+KE
+KI
+KP
+KR
+KW
+KG
+LA
+LV
+LB
+LS
+LR
+LY
+LI
+LT
+LU
+MO
+MK
+MG
+MW
+MY
+MV
+ML
+MT
+MH
+MQ
+MR
+MU
+YT
+MX
+FM
+MD
+MC
+MN
+MS
+MA
+MZ
+MM
+NA
+NR
+NP
+NL
+AN
+NC
+NZ
+NI
+NE
+NG
+NU
+NF
+MP
+NO
+OM
+PK
+PW
+PA
+PG
+PY
+PE
+PH
+PN
+PL
+PT
+PR
+QA
+RE
+RO
+RU
+RW
+KN
+LC
+VC
+WS
+SM
+ST
+SA
+SN
+SC
+SL
+SG
+SK
+SI
+SB
+SO
+ZA
+GS
+ES
+LK
+SH
+PM
+SD
+SR
+SJ
+SZ
+SE
+CH
+SY
+TW
+TJ
+TZ
+TH
+TG
+TK
+TO
+TT
+TN
+TR
+TM
+TC
+TV
+UG
+UA
+AE
+GB
+US
+UM
+UY
+UZ
+VU
+VE
+VN
+VG
+VI
+WF
+EH
+YE
+ZM
+ZW
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/43d71f10/sandbox/synth-log/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/sandbox/synth-log/src/main/resources/log4j.properties b/sandbox/synth-log/src/main/resources/log4j.properties
new file mode 100644
index 0000000..b445615
--- /dev/null
+++ b/sandbox/synth-log/src/main/resources/log4j.properties
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set root logger level to DEBUG and its only appender to A1.
+log4j.rootLogger=DEBUG, A1
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+
+# A1 uses PatternLayout.
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+
+log4j.logger.org.apache=off