You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@vxquery.apache.org by pr...@apache.org on 2014/07/16 23:54:50 UTC
git commit: The next round of benchmark test updates.
Repository: incubator-vxquery
Updated Branches:
refs/heads/prestonc/benchmark_updates 1b7d3aafe -> 2d6da4e23
The next round of benchmark test updates.
- Added a method to monitor cpu and io with a nice csv log.
- Updated the benchmark partitioner to use less file handlers.
- Buffer now is optional.
- Includes test to find out parsing and file read efficiency.
Project: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/commit/2d6da4e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/tree/2d6da4e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-vxquery/diff/2d6da4e2
Branch: refs/heads/prestonc/benchmark_updates
Commit: 2d6da4e23830864b5fbd449bbad0feda3c85ad66
Parents: 1b7d3aa
Author: Preston Carman <pr...@apache.org>
Authored: Wed Jul 16 14:54:37 2014 -0700
Committer: Preston Carman <pr...@apache.org>
Committed: Wed Jul 16 14:54:37 2014 -0700
----------------------------------------------------------------------
.../noaa-ghcn-daily/queries/no_result.xq | 24 ++++
.../noaa-ghcn-daily/scripts/weather_example.xml | 34 ------
.../scripts/weather_example_cluster.xml | 56 ---------
.../java/org/apache/vxquery/cli/VXQuery.java | 4 +-
.../org/apache/vxquery/xmlparser/XMLParser.java | 18 ++-
vxquery-xtest/pom.xml | 22 ++++
.../vxquery/xtest/util/DiskPerformance.java | 39 +++---
.../xtest/util/tests/AbstractDiskTest.java | 44 ++++---
.../tests/BufferedParsedCharacterStream.java | 2 +-
.../tests/BufferedReaderBufferedStream.java | 9 +-
.../xtest/util/tests/BufferedReaderStream.java | 9 +-
.../xtest/util/tests/BufferedStream.java | 9 +-
.../vxquery/xtest/util/tests/IDiskTest.java | 4 +-
.../util/tests/ParsedBufferedByteStream.java | 2 +-
.../tests/ParsedBufferedCharacterStream.java | 2 +-
.../xtest/util/tests/ParsedByteStream.java | 2 +-
.../xtest/util/tests/ParsedCharacterStream.java | 2 +-
.../xtest/util/tests/ReaderBufferedStream.java | 9 +-
.../vxquery/xtest/util/tests/ReaderStream.java | 9 +-
.../apache/vxquery/xtest/util/tests/Stream.java | 9 +-
.../resources/scripts/build_thread_summary.py | 121 +++++++++++++++++++
.../main/resources/scripts/clear_file_cache.sh | 18 +++
.../main/resources/scripts/run_thread_test.sh | 67 ++++++++++
23 files changed, 358 insertions(+), 157 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/no_result.xq
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/no_result.xq b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/no_result.xq
new file mode 100644
index 0000000..c1363e3
--- /dev/null
+++ b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/queries/no_result.xq
@@ -0,0 +1,24 @@
+(: Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License. :)
+
+(:
+VXQuery used to only parse all files with out producing results.
+:)
+let $sensor_collection := "/tmp/1.0_partition_ghcnd_all_xml/sensors"
+for $r in collection($sensor_collection)/dataCollection/data
+where fn:false()
+return $r
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
deleted file mode 100644
index 94c1440..0000000
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<data xmlns="data">
- <name>Local Example</name>
- <save_path>/data</save_path>
- <package>ghcnd_all</package>
- <node>
- <id>localhost</id>
- <cluster_ip>127.0.0.1</cluster_ip>
- </node>
- <dataset>
- <name>tiny-example</name>
- <test>local_speed_up</test>
- <save_path>/data</save_path>
- <partitions_per_path>1</partitions_per_path>
- <partitions_per_path>2</partitions_per_path>
- <partitions_per_path>4</partitions_per_path>
- <partitions_per_path>8</partitions_per_path>
- </dataset>
-</data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example_cluster.xml
----------------------------------------------------------------------
diff --git a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example_cluster.xml b/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example_cluster.xml
deleted file mode 100644
index 6078627..0000000
--- a/vxquery-benchmark/src/main/resources/noaa-ghcn-daily/scripts/weather_example_cluster.xml
+++ /dev/null
@@ -1,56 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<data xmlns="data">
- <name>Cluster Example</name>
- <save_path>/data</save_path>
- <package>ghcnd_all</package>
- <node>
- <id>machine1</id>
- <cluster_ip>127.0.0.1</cluster_ip>
- </node>
- <node>
- <id>machine2</id>
- <cluster_ip>127.0.0.2</cluster_ip>
- </node>
- <node>
- <id>machine3</id>
- <cluster_ip>127.0.0.3</cluster_ip>
- </node>
- <node>
- <id>machine4</id>
- <cluster_ip>127.0.0.4</cluster_ip>
- </node>
- <node>
- <id>machine5</id>
- <cluster_ip>127.0.0.5</cluster_ip>
- </node>
- <dataset>
- <name>tiny-1drive</name>
- <test>speed_up</test>
- <test>batch_scale_out</test>
- <save_path>/data</save_path>
- <partitions_per_path>1</partitions_per_path>
- </dataset>
- <dataset>
- <name>small-2drives</name>
- <test>speed_up</test>
- <test>batch_scale_out</test>
- <save_path>/data</save_path>
- <save_path>/data2</save_path>
- <partitions_per_path>1</partitions_per_path>
- </dataset>
-</data>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
----------------------------------------------------------------------
diff --git a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
index 61226e5..a0a5c5a 100644
--- a/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
+++ b/vxquery-cli/src/main/java/org/apache/vxquery/cli/VXQuery.java
@@ -448,8 +448,8 @@ public class VXQuery {
@Option(name = "-frame-size", usage = "Frame size in bytes. (default 65536)")
public int frameSize = 65536;
- @Option(name = "-buffer-size", usage = "Read file buffer size in bytes. (default 512)")
- public int bufferSize = 512;
+ @Option(name = "-buffer-size", usage = "Disk read buffer size in bytes.")
+ public int bufferSize = -1;
@Option(name = "-O", usage = "Optimization Level. Default: Full Optimization")
private int optimizationLevel = Integer.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
----------------------------------------------------------------------
diff --git a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
index 1aea1e6..1d979b5 100644
--- a/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
+++ b/vxquery-core/src/main/java/org/apache/vxquery/xmlparser/XMLParser.java
@@ -44,7 +44,7 @@ public class XMLParser {
final SAXContentHandler handler;
final InputSource in;
final String nodeId;
- final int buffer_size;
+ final int bufferSize;
public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider, String nodeId) throws HyracksDataException {
this(attachTypes, idProvider, nodeId, null, null, null, null);
@@ -53,12 +53,10 @@ public class XMLParser {
public XMLParser(boolean attachTypes, ITreeNodeIdProvider idProvider, String nodeId, ByteBuffer frame,
FrameTupleAppender appender, List<Integer> childSeq, StaticContext staticContext)
throws HyracksDataException {
- buffer_size = Integer.parseInt(System.getProperty("vxquery.buffer_size"));
+ bufferSize = Integer.parseInt(System.getProperty("vxquery.buffer_size"));
this.nodeId = nodeId;
try {
parser = XMLReaderFactory.createXMLReader();
-
- System.out.println("XMLReader buffer:" +parser.getProperty("http://apache.org/xml/properties/input-buffer-size"));
if (frame == null || appender == null) {
handler = new SAXContentHandler(attachTypes, idProvider);
} else {
@@ -78,7 +76,11 @@ public class XMLParser {
public void parseDocument(File file, ArrayBackedValueStorage abvs) throws HyracksDataException {
try {
- in.setCharacterStream(new BufferedReader(new InputStreamReader(new FileInputStream(file)), buffer_size));
+ if (bufferSize > 0) {
+ in.setCharacterStream(new BufferedReader(new InputStreamReader(new FileInputStream(file)), bufferSize));
+ } else {
+ in.setCharacterStream(new InputStreamReader(new FileInputStream(file)));
+ }
parser.parse(in);
handler.writeDocument(abvs);
} catch (FileNotFoundException e) {
@@ -99,7 +101,11 @@ public class XMLParser {
public void parseElements(File file, IFrameWriter writer, FrameTupleAccessor fta, int tupleIndex)
throws HyracksDataException {
try {
- in.setCharacterStream(new BufferedReader(new InputStreamReader(new FileInputStream(file)), buffer_size));
+ if (bufferSize > 0) {
+ in.setCharacterStream(new BufferedReader(new InputStreamReader(new FileInputStream(file)), bufferSize));
+ } else {
+ in.setCharacterStream(new InputStreamReader(new FileInputStream(file)));
+ }
handler.setupElementWriter(writer, fta, tupleIndex);
parser.parse(in);
} catch (FileNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/pom.xml
----------------------------------------------------------------------
diff --git a/vxquery-xtest/pom.xml b/vxquery-xtest/pom.xml
index ee79b2a..320950b 100644
--- a/vxquery-xtest/pom.xml
+++ b/vxquery-xtest/pom.xml
@@ -65,6 +65,28 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>2.5</version>
+ <executions>
+ <execution>
+ <id>copy-scripts</id>
+ <!-- here the phase you need -->
+ <phase>package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>target/appassembler/bin</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources/scripts</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<!--
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/DiskPerformance.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/DiskPerformance.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/DiskPerformance.java
index 913d22e..dd43de7 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/DiskPerformance.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/DiskPerformance.java
@@ -99,11 +99,11 @@ public class DiskPerformance {
testFilesIt = cTestFiles.iterator();
}
- public String getNextFile() {
+ public File getNextFile() {
if (!testFilesIt.hasNext()) {
testFilesIt = cTestFiles.iterator();
}
- return testFilesIt.next().getAbsolutePath();
+ return testFilesIt.next();
}
private byte[] createUTF8String(String str) {
@@ -145,19 +145,21 @@ public class DiskPerformance {
dp.setDirectory(args[0]);
ArrayList<Class> tests = new ArrayList<Class>();
- // Character Streams
+ // Parsed Character Streams
tests.add(ParsedBufferedCharacterStream.class);
- tests.add(BufferedParsedCharacterStream.class);
- tests.add(ParsedCharacterStream.class);
- tests.add(BufferedReaderBufferedStream.class);
- tests.add(BufferedReaderStream.class);
- tests.add(ReaderBufferedStream.class);
- tests.add(ReaderStream.class);
+ // tests.add(BufferedParsedCharacterStream.class);
+ // tests.add(ParsedCharacterStream.class);
+ // Parsed Byte Streams
+ // tests.add(ParsedBufferedByteStream.class);
+ // tests.add(ParsedByteStream.class);
+ // Character Streams
+ // tests.add(BufferedReaderBufferedStream.class);
+ // tests.add(BufferedReaderStream.class);
+ // tests.add(ReaderBufferedStream.class);
+ // tests.add(ReaderStream.class);
// Byte Streams
- tests.add(ParsedBufferedByteStream.class);
- tests.add(ParsedByteStream.class);
- tests.add(BufferedStream.class);
- tests.add(Stream.class);
+ // tests.add(BufferedStream.class);
+ // tests.add(Stream.class);
System.out.println("------");
System.out.println("Started Test Group: " + new Date());
@@ -173,7 +175,7 @@ public class DiskPerformance {
runThreadTest(testClass, dp, threads, bufferSize);
} else {
IDiskTest test = testClass.newInstance();
- test.setFilename(dp.getNextFile());
+ test.setFile(dp.getNextFile());
test.setBufferSize(bufferSize);
test.setParser(dp.parser);
test.run();
@@ -185,20 +187,19 @@ public class DiskPerformance {
}
}
- System.out.println("------");
}
}
- static <T> void runThreadTest(Class<T> testType, DiskPerformance dp, int threads, int bufferSize) throws InstantiationException,
- IllegalAccessException {
+ static <T> void runThreadTest(Class<T> testType, DiskPerformance dp, int threads, int bufferSize)
+ throws InstantiationException, IllegalAccessException {
ExecutorService es = Executors.newCachedThreadPool();
ArrayList<IDiskTest> threadTests = new ArrayList<IDiskTest>();
for (int i = 0; i < threads; ++i) {
threadTests.add((IDiskTest) testType.newInstance());
}
for (IDiskTest test : threadTests) {
- test.setFilename(dp.getNextFile());
+ test.setFile(dp.getNextFile());
test.setBufferSize(bufferSize);
test.setParser(dp.getNewParser().first);
es.execute((Runnable) test);
@@ -209,6 +210,6 @@ public class DiskPerformance {
} catch (InterruptedException e) {
e.printStackTrace();
}
- System.out.println("Completed thread batch: " + new Date());
+// System.out.println("Completed thread batch: " + new Date());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/AbstractDiskTest.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/AbstractDiskTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/AbstractDiskTest.java
index 63d5922..f41a02b 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/AbstractDiskTest.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/AbstractDiskTest.java
@@ -14,9 +14,10 @@
*/
package org.apache.vxquery.xtest.util.tests;
+import java.io.File;
import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.OperatingSystemMXBean;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import org.xml.sax.XMLReader;
@@ -25,16 +26,12 @@ abstract class AbstractDiskTest implements IDiskTest, Runnable {
private int bufferSize;
protected XMLReader parser;
- public AbstractDiskTest() {
+ public void setFile(File file) {
+ this.filename = file.getAbsolutePath();
}
- public AbstractDiskTest(String filename, int bufferSize) {
- setFilename(filename);
- setBufferSize(bufferSize);
- }
-
- public void setFilename(String filename) {
- this.filename = filename;
+ public String getPrintFilename() {
+ return filename.substring(filename.length() - 20);
}
public void setBufferSize(int bufferSize) {
@@ -50,22 +47,37 @@ abstract class AbstractDiskTest implements IDiskTest, Runnable {
}
public void run() {
- OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
- System.out.printf(filename + "\t" + getMessage() + " - Starting%n");
+ long size = -1;
+ // System.out.printf(getPrintFilename() + "\t" + getMessage() + " - Starting%n");
long start = System.nanoTime();
try {
- long checkSum = test(filename, bufferSize);
+ size = test(filename, bufferSize);
} catch (IOException e) {
e.printStackTrace();
}
long end = System.nanoTime();
+ long timeDelta = end - start;
+ double speed = 0;
+ if (size > 0) {
+ speed = (size * 1e3) / timeDelta;
+ }
// System.out.printf("checkSum: %d%n", checkSum);
- System.out.printf(filename + "\t" + getMessage() + "%.2f MB/s", 1024 * 1e9 / (end - start));
+ // System.out.printf(getPrintFilename() + "\t" + getMessage() + "%.1f ms\t%.2f MB/s\t%.2f MB/s",
+ // (timeDelta) / 1e6, 1024 * 1024 * 1e6 / (timeDelta), speed);
+ // CSV output of the results.
+ try {
+ System.out.printf("%s,", InetAddress.getLocalHost().getHostName());
+ } catch (UnknownHostException e) {
+ e.printStackTrace();
+ }
+ System.out.printf("%s,%s,%.1f,%.2f", filename, getMessage(), (timeDelta) / 1e6,
+ 1024 * 1024 * 1e6 / (timeDelta), speed);
if (isBuffered() && bufferSize > 0) {
- System.out.printf("\t%.1f KB buffer", bufferSize / 1024.0);
+ System.out.printf(",%.1f", bufferSize / 1024.0);
+ } else {
+ System.out.printf(",0");
}
System.out.println();
- System.out.printf("%.2f%% load average last minute%n", os.getSystemLoadAverage());
}
public boolean isBuffered() {
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedParsedCharacterStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedParsedCharacterStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedParsedCharacterStream.java
index 37527ae..8a9ee18 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedParsedCharacterStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedParsedCharacterStream.java
@@ -24,7 +24,7 @@ import org.xml.sax.SAXException;
public class BufferedParsedCharacterStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Buffered Parsed - Character Stream\t";
+ return "Buffered Parsed - Character Stream";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderBufferedStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderBufferedStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderBufferedStream.java
index c553a3e..d964241 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderBufferedStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderBufferedStream.java
@@ -23,7 +23,7 @@ import java.io.InputStreamReader;
public class BufferedReaderBufferedStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Buffered Reader - Buffered Stream\t";
+ return "Buffered Reader - Buffered Stream";
}
@Override
@@ -36,10 +36,13 @@ public class BufferedReaderBufferedStream extends AbstractDiskTest {
f = new BufferedReader(new InputStreamReader(new BufferedInputStream(new FileInputStream(filename))));
}
int b;
+ long byteCount = 0L;
long checkSum = 0L;
- while ((b = f.read()) != -1)
+ while ((b = f.read()) != -1) {
checkSum += b;
+ byteCount++;
+ }
f.close();
- return checkSum;
+ return byteCount;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderStream.java
index 20b9bab..7100025 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedReaderStream.java
@@ -22,7 +22,7 @@ import java.io.InputStreamReader;
public class BufferedReaderStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Buffered Reader - Stream\t\t";
+ return "Buffered Reader - Stream";
}
@Override
@@ -34,10 +34,13 @@ public class BufferedReaderStream extends AbstractDiskTest {
f = new BufferedReader(new InputStreamReader(new FileInputStream(filename)));
}
int b;
+ long byteCount = 0L;
long checkSum = 0L;
- while ((b = f.read()) != -1)
+ while ((b = f.read()) != -1) {
checkSum += b;
+ byteCount++;
+ }
f.close();
- return checkSum;
+ return byteCount;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedStream.java
index 2bf582e..5751307 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/BufferedStream.java
@@ -21,7 +21,7 @@ import java.io.IOException;
public class BufferedStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Buffered Stream\t\t\t\t";
+ return "Buffered Stream";
}
@Override
@@ -33,10 +33,13 @@ public class BufferedStream extends AbstractDiskTest {
f = new BufferedInputStream(new FileInputStream(filename));
}
int b;
+ long byteCount = 0L;
long checkSum = 0L;
- while ((b = f.read()) != -1)
+ while ((b = f.read()) != -1) {
checkSum += b;
+ byteCount++;
+ }
f.close();
- return checkSum;
+ return byteCount;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/IDiskTest.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/IDiskTest.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/IDiskTest.java
index 09ff240..4c56710 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/IDiskTest.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/IDiskTest.java
@@ -1,11 +1,13 @@
package org.apache.vxquery.xtest.util.tests;
+import java.io.File;
+
import org.xml.sax.XMLReader;
public interface IDiskTest {
public void setBufferSize(int bufferSize);
- public void setFilename(String absolutePath);
+ public void setFile(File file);
public void setParser(XMLReader parser);
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedByteStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedByteStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedByteStream.java
index 5ae2a34..9fc93c4 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedByteStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedByteStream.java
@@ -24,7 +24,7 @@ import org.xml.sax.SAXException;
public class ParsedBufferedByteStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Parsed - Buffered Byte Stream\t\t";
+ return "Parsed - Buffered Byte Stream";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedCharacterStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedCharacterStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedCharacterStream.java
index c2d21d4..1cacee6 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedCharacterStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedBufferedCharacterStream.java
@@ -25,7 +25,7 @@ import org.xml.sax.SAXException;
public class ParsedBufferedCharacterStream extends AbstractDiskTest implements IDiskTest {
@Override
public String getMessage() {
- return "Parsed - Buffered Character Stream\t";
+ return "Parsed - Buffered Character Stream";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedByteStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedByteStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedByteStream.java
index 2c8b85d..783424a 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedByteStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedByteStream.java
@@ -23,7 +23,7 @@ import org.xml.sax.SAXException;
public class ParsedByteStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Parsed - Byte Stream\t\t\t";
+ return "Parsed - Byte Stream";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedCharacterStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedCharacterStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedCharacterStream.java
index 3dc9ed4..0b292a3 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedCharacterStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ParsedCharacterStream.java
@@ -24,7 +24,7 @@ import org.xml.sax.SAXException;
public class ParsedCharacterStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Parsed - Character Stream\t\t";
+ return "Parsed - Character Stream";
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderBufferedStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderBufferedStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderBufferedStream.java
index 3d87f96..721dd3a 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderBufferedStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderBufferedStream.java
@@ -22,7 +22,7 @@ import java.io.InputStreamReader;
public class ReaderBufferedStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Reader - Buffered Stream\t\t";
+ return "Reader - Buffered Stream";
}
@Override
@@ -34,10 +34,13 @@ public class ReaderBufferedStream extends AbstractDiskTest {
f = new InputStreamReader(new BufferedInputStream(new FileInputStream(filename)));
}
int b;
+ long byteCount = 0L;
long checkSum = 0L;
- while ((b = f.read()) != -1)
+ while ((b = f.read()) != -1) {
checkSum += b;
+ byteCount++;
+ }
f.close();
- return checkSum;
+ return byteCount;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderStream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderStream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderStream.java
index 2a01275..4fdbdf9 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderStream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/ReaderStream.java
@@ -21,7 +21,7 @@ import java.io.InputStreamReader;
public class ReaderStream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Reader - Stream\t\t\t\t";
+ return "Reader - Stream";
}
@Override
@@ -33,10 +33,13 @@ public class ReaderStream extends AbstractDiskTest {
public long test(String filename, int bufferSize) throws IOException {
InputStreamReader f = new InputStreamReader(new FileInputStream(filename));
int b;
+ long byteCount = 0L;
long checkSum = 0L;
- while ((b = f.read()) != -1)
+ while ((b = f.read()) != -1) {
checkSum += b;
+ byteCount++;
+ }
f.close();
- return checkSum;
+ return byteCount;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/Stream.java
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/Stream.java b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/Stream.java
index 96e7184..7890b2a 100644
--- a/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/Stream.java
+++ b/vxquery-xtest/src/main/java/org/apache/vxquery/xtest/util/tests/Stream.java
@@ -20,7 +20,7 @@ import java.io.IOException;
public class Stream extends AbstractDiskTest {
@Override
public String getMessage() {
- return "Stream\t\t\t\t\t\t";
+ return "Stream";
}
@Override
@@ -32,10 +32,13 @@ public class Stream extends AbstractDiskTest {
public long test(String filename, int bufferSize) throws IOException {
FileInputStream f = new FileInputStream(filename);
int b;
+ long byteCount = 0L;
long checkSum = 0L;
- while ((b = f.read()) != -1)
+ while ((b = f.read()) != -1) {
checkSum += b;
+ byteCount++;
+ }
f.close();
- return checkSum;
+ return byteCount;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/resources/scripts/build_thread_summary.py
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/resources/scripts/build_thread_summary.py b/vxquery-xtest/src/main/resources/scripts/build_thread_summary.py
new file mode 100644
index 0000000..835b2d2
--- /dev/null
+++ b/vxquery-xtest/src/main/resources/scripts/build_thread_summary.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import fnmatch
+import getopt
+import glob
+import os
+import sys
+import csv
+
+SEARCH_STRING = 'Average execution time:'
+
+def find_files(directory, pattern):
+ for root, dirs, files in os.walk(directory):
+ for basename in files:
+ if fnmatch.fnmatch(basename, pattern):
+ yield (root, basename)
+
+
+def main(argv):
+ ''' Same as bash: find $FOLDER -type f -name "*.xml" -exec basename {} \; > list_xml.csv
+ '''
+ log_folder = ""
+ save_file = ""
+
+ fields = ("usr", "wai", "read")
+ fields_indexes = (0, 3, 6)
+ fields_conversion = (1, 1, 1024*1024)
+ fields_min = [999, 999, 999]
+ fields_max = [0, 0, 0]
+ fields_sum = [0, 0, 0]
+ poll_count = 0
+
+ START_LINE = 7
+
+ # Get the base folder
+ try:
+ opts, args = getopt.getopt(argv, "f:hs:t:", ["folder=", "save_file="])
+ except getopt.GetoptError:
+ print 'The file options for list_xml_files.py were not correctly specified.'
+ print 'To see a full list of options try:'
+ print ' $ python list_xml_files.py -h'
+ sys.exit(2)
+ for opt, arg in opts:
+ if opt == '-h':
+ print 'Options:'
+ print ' -f The base folder to build XML file list.'
+ print ' -s The save file.'
+ sys.exit()
+ elif opt in ('-f', "--folder"):
+ # check if file exists.
+ if os.path.exists(arg):
+ log_folder = arg
+ else:
+ print 'Error: Argument must be a folder name for --folder (-f).'
+ sys.exit()
+ elif opt in ('-s', "--save_file"):
+ save_file = arg
+
+ # Required fields to run the script.
+ if log_folder == "" or not os.path.exists(log_folder):
+ print 'Error: The folder path option must be supplied: --folder (-f).'
+ sys.exit()
+ if save_file == "":
+ print 'Error: The folder path option must be supplied: --save_file (-s).'
+ sys.exit()
+
+ with open(save_file, 'w') as outfile:
+ csvfile = csv.writer(outfile)
+ for path, filename in find_files(log_folder, '*thread_*.csv'):
+ keys = filename.split(".")[0].split("_")
+ with open(path + "/" + filename, 'rU') as infile:
+ csv_input_file = csv.reader(infile, quotechar='"', delimiter = ',')
+ count = 0
+ print 'Working on file: ' + filename
+ for line in csv_input_file:
+ if (count > START_LINE):
+ for i, v in enumerate(fields_indexes):
+ if (len(line) <= v):
+ print 'Bad file: ' + filename
+ break
+ else:
+ if (float(line[v]) < fields_min[i]):
+ fields_min[i] = float(line[v])
+ if (fields_max[i] < float(line[v])):
+ fields_max[i] = float(line[v])
+ fields_sum[i] += float(line[v])
+ poll_count += 1
+ count += 1
+
+ # Build data row
+ row = []
+ if (poll_count > 0):
+ row.extend(keys)
+ for i, v in enumerate(fields_indexes):
+ row.append("%.2f" % (fields_min[i] / fields_conversion[i]))
+ row.append("%.2f" % (fields_max[i] / fields_conversion[i]))
+ row.append("%.2f" % (fields_sum[i] / fields_conversion[i] / poll_count))
+ csvfile.writerow(row)
+
+ # Reset
+ fields_min = [999, 999, 999]
+ fields_max = [0, 0, 0]
+ fields_sum = [0, 0, 0]
+ poll_count = 0
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/resources/scripts/clear_file_cache.sh
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/resources/scripts/clear_file_cache.sh b/vxquery-xtest/src/main/resources/scripts/clear_file_cache.sh
new file mode 100644
index 0000000..b5ef375
--- /dev/null
+++ b/vxquery-xtest/src/main/resources/scripts/clear_file_cache.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+BYTE_COUNT=2000000
+
+# loop over the files.
+for i in `seq 0 9`
+do
+ # Attempting to add files to the cache.
+ echo "cache file: /tmp/zero${i}"
+ for l in `seq 0 4`
+ do
+ if [ ! -f "/tmp/zero${i}" ]
+ then
+ echo "Creating..."
+ dd if=/dev/zero of=/tmp/zero${i} count=${BYTE_COUNT} 2> /dev/null
+ fi;
+ dd if=/tmp/zero${i} of=/dev/null count=${BYTE_COUNT} 2> /dev/null
+ done;
+done;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-vxquery/blob/2d6da4e2/vxquery-xtest/src/main/resources/scripts/run_thread_test.sh
----------------------------------------------------------------------
diff --git a/vxquery-xtest/src/main/resources/scripts/run_thread_test.sh b/vxquery-xtest/src/main/resources/scripts/run_thread_test.sh
new file mode 100644
index 0000000..30e197c
--- /dev/null
+++ b/vxquery-xtest/src/main/resources/scripts/run_thread_test.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+
+if [ -z "${1}" ]
+then
+ echo "Please supply a directory for XML files."
+ exit
+fi
+
+
+BUFFER=$((8*1024))
+# bash until loop
+until [ ${BUFFER} -gt $((1024*1024*1024)) ]; do
+ for n in 1 2 4
+ do
+ # Parse Only
+ log_file="thread_parsed_${n}_${BUFFER}.csv"
+
+ echo "Clearing file system cache. (hack)"
+ sh ./vxquery-xtest/target/appassembler/bin/clear_file_cache.sh > /dev/null
+
+ echo "Start background logging."
+ dstat -cdmr --nocolor --output logs/${log_file} > /dev/null &
+ PROC_ID=$!
+
+ echo "Run experiment for ${n} threads."
+ JAVA_OPTS=" -Xmx8G " sh ./vxquery-xtest/target/appassembler/bin/diskperformance ${1} ${n} 1 ${BUFFER}
+
+ echo "Stop background logging. (Process ID: ${PROC_ID})"
+ kill -9 ${PROC_ID}
+
+
+ # Parse No Return
+ log_file="thread_empty_${n}_${BUFFER}.csv"
+
+ echo "Clearing file system cache. (hack)"
+ sh ./vxquery-xtest/target/appassembler/bin/clear_file_cache.sh > /dev/null
+
+ echo "Start background logging."
+ dstat -cdmr --nocolor --output logs/${log_file} > /dev/null &
+ PROC_ID=$!
+
+ echo "Run experiment for ${n} threads."
+ JAVA_OPTS=" -Xmx8G " sh ./vxquery-cli/target/appassembler/bin/vxq weather_data/dataset-gsn-local/queries/local_batch_scale_out/d1_p${n}/no_result.xq -buffer-size ${BUFFER} 2>&1
+
+ echo "Stop background logging. (Process ID: ${PROC_ID})"
+ kill -9 ${PROC_ID}
+
+ # Full Query
+ log_file="thread_full_${n}_${BUFFER}.csv"
+
+ echo "Clearing file system cache. (hack)"
+ sh ./vxquery-xtest/target/appassembler/bin/clear_file_cache.sh > /dev/null
+
+ echo "Start background logging."
+ dstat -cdmr --nocolor --output logs/${log_file} > /dev/null &
+ PROC_ID=$!
+
+ echo "Run experiment for ${n} threads."
+ JAVA_OPTS=" -Xmx8G " sh ./vxquery-cli/target/appassembler/bin/vxq weather_data/dataset-gsn-local/queries/local_batch_scale_out/d1_p${n}/q00.xq -buffer-size ${BUFFER} 2>&1
+
+ echo "Stop background logging. (Process ID: ${PROC_ID})"
+ kill -9 ${PROC_ID}
+
+ done;
+ let BUFFER=BUFFER*4
+done