You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2017/11/30 04:55:53 UTC
[08/14] asterixdb git commit: [NO ISSUE] Delete asterix-experiments
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java
deleted file mode 100644
index c046f4e..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGenerator.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.client;
-
-import java.io.BufferedReader;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.Inet4Address;
-import java.net.Socket;
-import java.util.Random;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.experiment.action.base.IAction;
-import org.apache.asterix.experiment.action.base.SequentialActionList;
-import org.apache.asterix.experiment.action.derived.RunAQLStringAction;
-import org.apache.asterix.experiment.action.derived.TimedAction;
-import org.apache.http.client.HttpClient;
-import org.apache.http.impl.client.DefaultHttpClient;
-import org.apache.hyracks.api.util.ExperimentProfilerUtils;
-
-public class SpatialQueryGenerator {
-
- private static final int SKIP_LINE_COUNT = 199;
-
- private final ExecutorService threadPool;
-
- private final int partitionRangeStart;
-
- private final int duration;
-
- private final String restHost;
-
- private final int restPort;
-
- private final String orchHost;
-
- private final int orchPort;
-
- private final String openStreetMapFilePath;
-
- private final boolean isIndexOnlyPlan;
-
- public SpatialQueryGenerator(SpatialQueryGeneratorConfig config) {
- threadPool = Executors.newCachedThreadPool(new ThreadFactory() {
-
- private final AtomicInteger count = new AtomicInteger();
-
- @Override
- public Thread newThread(Runnable r) {
- int tid = count.getAndIncrement();
- Thread t = new Thread(r, "DataGeneratorThread: " + tid);
- t.setDaemon(true);
- return t;
- }
- });
-
- partitionRangeStart = config.getPartitionRangeStart();
- duration = config.getDuration();
- restHost = config.getRESTHost();
- restPort = config.getRESTPort();
- orchHost = config.getQueryOrchestratorHost();
- orchPort = config.getQueryOrchestratorPort();
- openStreetMapFilePath = config.getOpenStreetMapFilePath();
- isIndexOnlyPlan = config.getIsIndexOnlyPlan();
- }
-
- public void start() throws Exception {
- final Semaphore sem = new Semaphore(0);
- threadPool.submit(new QueryGenerator(sem, restHost, restPort, orchHost, orchPort, partitionRangeStart,
- duration, openStreetMapFilePath, isIndexOnlyPlan));
- sem.acquire();
- }
-
- public static class QueryGenerator implements Runnable {
-
- private static final Logger LOGGER = Logger.getLogger(QueryGenerator.class.getName());
-
- private final HttpClient httpClient;
- private final Semaphore sem;
- private final String restHost;
- private final int restPort;
- private final String orchHost;
- private final int orchPort;
- private final int partition;
- private final int queryDuration;
- private final String openStreetMapFilePath;
- private final float[] radiusType = new float[] { 0.00001f, 0.0001f, 0.001f, 0.01f, 0.1f };
- private int radiusIter = 0;
- private final Random randGen;
- private BufferedReader br;
- private long queryCount;
- private Random random = new Random(211);
- private final boolean isIndexOnlyPlan;
- private String outputFilePath;
- private FileOutputStream outputFos;
-
- public QueryGenerator(Semaphore sem, String restHost, int restPort, String orchHost, int orchPort,
- int partitionRangeStart, int queryDuration, String openStreetMapFilePath, boolean isIndexOnlyPlan) {
- httpClient = new DefaultHttpClient();
- this.sem = sem;
- this.restHost = restHost;
- this.restPort = restPort;
- this.orchHost = orchHost;
- this.orchPort = orchPort;
- this.partition = partitionRangeStart;
- this.queryDuration = queryDuration * 1000;
- this.openStreetMapFilePath = openStreetMapFilePath;
- this.queryCount = 0;
- this.randGen = new Random(partitionRangeStart);
- this.isIndexOnlyPlan = isIndexOnlyPlan;
- }
-
- @Override
- public void run() {
- LOGGER.info("\nQueryGen[" + partition + "] running with the following parameters: \n"
- + "queryGenDuration : " + queryDuration + "\n" + "isIndexOnlyPlan : " + isIndexOnlyPlan);
-
- try {
- outputFilePath = openStreetMapFilePath.substring(0, openStreetMapFilePath.lastIndexOf(File.separator))
- + File.separator + "QueryGenResult-" + Inet4Address.getLocalHost().getHostAddress() + ".txt";
- outputFos = ExperimentProfilerUtils.openOutputFile(outputFilePath);
- } catch (Exception e) {
- e.printStackTrace();
- return;
- }
-
- try {
- if (openStreetMapFilePath != null) {
- this.br = new BufferedReader(new FileReader(openStreetMapFilePath));
- }
- try {
- //connect to orchestrator socket
- Socket orchSocket = null;
- orchSocket = new Socket(orchHost, orchPort);
- try {
- //send reached message to orchestrator
- sendReached(orchSocket);
-
- //wait until receiving resume message from the orchestrator server
- receiveResume(orchSocket);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("QueryGen[" + partition + "] starts sending queries...");
- }
- //send queries during query duration
- long startTS = System.currentTimeMillis();
- long prevTS = startTS;
- long curTS = startTS;
- while (curTS - startTS < queryDuration) {
- sendQuery();
- queryCount++;
- curTS = System.currentTimeMillis();
- if (LOGGER.isLoggable(Level.INFO) && queryCount % 100 == 0) {
- LOGGER.info("QueryGen[" + partition + "][TimeToQuery100] " + (curTS - prevTS)
- + " in milliseconds");
- prevTS = curTS;
- }
- }
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("QueryGen[" + partition + "][QueryCount] " + queryCount + " in "
- + (queryDuration / 1000) + " seconds");
- }
-
- if (outputFos != null) {
- ExperimentProfilerUtils.closeOutputFile(outputFos);
- }
-
- //send reqched message to orchestrator
- sendReached(orchSocket);
- } finally {
- if (orchSocket != null) {
- orchSocket.close();
- }
- }
- } catch (Throwable t) {
- t.printStackTrace();
- outputFos.write("Error during sending query\n".getBytes());
- throw t;
- } finally {
- if (openStreetMapFilePath != null) {
- br.close();
- }
- if (outputFos != null) {
- ExperimentProfilerUtils.closeOutputFile(outputFos);
- }
- }
- } catch (Throwable t) {
- System.err.println("Error connecting to rest API server " + restHost + ":" + restPort);
- t.printStackTrace();
- } finally {
- sem.release();
- }
- }
-
- private void sendQuery() throws IOException {
- //prepare radius and center point
- int skipLineCount = SKIP_LINE_COUNT;
- int lineCount = 0;
- String line = null;;
- String strPoints[] = null;
- float x = 0, y = 0;
- int beginX = -180, endX = 180, beginY = -90, endY = 90;
- if (openStreetMapFilePath != null) {
- while (lineCount < skipLineCount) {
- if ((line = br.readLine()) == null) {
- //reopen file
- br.close();
- br = new BufferedReader(new FileReader(openStreetMapFilePath));
- line = br.readLine();
- }
- strPoints = line.split(",");
- if (strPoints.length != 2) {
- continue;
- }
- lineCount++;
- }
- y = Float.parseFloat(strPoints[0]) / 10000000; //latitude (y value)
- x = Float.parseFloat(strPoints[1]) / 10000000; //longitude (x value)
- } else {
- int xMajor = beginX + random.nextInt(endX - beginX);
- int xMinor = random.nextInt(100);
- x = xMajor + ((float) xMinor) / 100;
-
- int yMajor = beginY + random.nextInt(endY - beginY);
- int yMinor = random.nextInt(100);
- y = yMajor + ((float) yMinor) / 100;
- }
-
- //create action
- SequentialActionList sAction = new SequentialActionList();
- IAction rangeQueryAction = new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort,
- getRangeQueryAQL(radiusType[radiusIter++ % radiusType.length], x, y, isIndexOnlyPlan), outputFos),
- outputFos);
- sAction.add(rangeQueryAction);
-
- //perform
- sAction.perform();
- }
-
- private String getRangeQueryAQL(float radius, float x, float y, boolean isIndexOnlyPlan) {
- StringBuilder sb = new StringBuilder();
- sb.append("use dataverse experiments; ");
- sb.append("count( ");
- sb.append("for $x in dataset Tweets").append(" ");
- sb.append("let $n := create-circle( ");
- sb.append("point(\"").append(x).append(", ").append(y).append("\") ");
- sb.append(", ");
- sb.append(String.format("%f", radius));
- sb.append(" )");
- if (isIndexOnlyPlan) {
- sb.append("where spatial-intersect($x.sender-location, $n) ");
- } else {
- sb.append("where spatial-intersect($x.sender-location, $n) and $x.btree-extra-field1 <= int32(\"2147483647\") ");
- }
- sb.append("return $x ");
- sb.append(");");
- return sb.toString();
- }
-
- private void sendReached(Socket s) throws IOException {
- new DataOutputStream(s.getOutputStream()).writeInt(OrchestratorDGProtocol.REACHED.ordinal());
- s.getOutputStream().flush();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Sent " + OrchestratorDGProtocol.REACHED + " to " + s.getRemoteSocketAddress());
- }
- }
-
- private void receiveResume(Socket s) throws IOException {
- int msg = new DataInputStream(s.getInputStream()).readInt();
- OrchestratorDGProtocol msgType = OrchestratorDGProtocol.values()[msg];
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Received " + msgType + " from " + s.getRemoteSocketAddress());
- }
- if (msgType != OrchestratorDGProtocol.RESUME) {
- throw new IllegalStateException("Unknown protocol message received: " + msgType);
- }
- }
-
- private void sendStopped(Socket s) throws IOException {
- new DataOutputStream(s.getOutputStream()).writeInt(OrchestratorDGProtocol.STOPPED.ordinal());
- s.getOutputStream().flush();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Sent " + OrchestratorDGProtocol.STOPPED + " to " + s.getRemoteSocketAddress());
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java
deleted file mode 100644
index 79055aa..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorConfig.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.client;
-
-import org.apache.commons.lang3.tuple.Pair;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-import org.kohsuke.args4j.OptionDef;
-import org.kohsuke.args4j.spi.OptionHandler;
-import org.kohsuke.args4j.spi.Parameters;
-import org.kohsuke.args4j.spi.Setter;
-
-public class SpatialQueryGeneratorConfig {
-
- @Option(name = "-p", aliases = "--partition-range-start", usage = "Starting partition number for the set of data generators (default = 0)")
- private int partitionRangeStart = 0;
-
- public int getPartitionRangeStart() {
- return partitionRangeStart;
- }
-
- @Option(name = "-qd", aliases = { "--duration" }, usage = "Duration in seconds to run guery generation")
- private int queryGenDuration = -1;
-
- public int getDuration() {
- return queryGenDuration;
- }
-
- @Option(name = "-qc", aliases = { "--query-count" }, usage = "The number of queries to generate")
- private int queryCount = -1;
-
- public int getQueryCount() {
- return queryCount;
- }
-
- @Option(name = "-rh", aliases = "--rest-host", usage = "Asterix REST API host address", required = true, metaVar = "HOST")
- private String restHost;
-
- public String getRESTHost() {
- return restHost;
- }
-
- @Option(name = "-rp", aliases = "--rest-port", usage = "Asterix REST API port", required = true, metaVar = "PORT")
- private int restPort;
-
- public int getRESTPort() {
- return restPort;
- }
-
- @Option(name = "-qoh", aliases = "--query-orchestrator-host", usage = "The host address of query orchestrator")
- private String queryOrchHost;
-
- public String getQueryOrchestratorHost() {
- return queryOrchHost;
- }
-
- @Option(name = "-qop", aliases = "--query-orchestrator-port", usage = "The port to be used for the orchestrator server of query orchestrator")
- private int queryOrchPort;
-
- public int getQueryOrchestratorPort() {
- return queryOrchPort;
- }
-
- @Option(name = "-of", aliases = "--openstreetmap-filepath", usage = "The open street map gps point data file path")
- private String openStreetMapFilePath;
-
- public String getOpenStreetMapFilePath() {
- return openStreetMapFilePath;
- }
-
- @Option(name = "-iop", aliases = "--index-only-plan", usage = "Indicator of whether index only plan or not")
- private boolean isIndexOnlyPlan;
-
- public boolean getIsIndexOnlyPlan() {
- return isIndexOnlyPlan;
- }
-
- public static class AddressOptionHandler extends OptionHandler<Pair<String, Integer>> {
-
- public AddressOptionHandler(CmdLineParser parser, OptionDef option, Setter<? super Pair<String, Integer>> setter) {
- super(parser, option, setter);
- }
-
- @Override
- public int parseArguments(Parameters params) throws CmdLineException {
- int counter = 0;
- while (true) {
- String param;
- try {
- param = params.getParameter(counter);
- } catch (CmdLineException ex) {
- break;
- }
-
- String[] hostPort = param.split(":");
- if (hostPort.length != 2) {
- throw new CmdLineException("Invalid address: " + param + ". Expected <host>:<port>");
- }
- Integer port = null;
- try {
- port = Integer.parseInt(hostPort[1]);
- } catch (NumberFormatException e) {
- throw new CmdLineException("Invalid port " + hostPort[1] + " for address " + param + ".");
- }
- setter.addValue(Pair.of(hostPort[0], port));
- counter++;
- }
- return counter;
- }
-
- @Override
- public String getDefaultMetaVariable() {
- return "addresses";
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java
deleted file mode 100644
index ee16340..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SpatialQueryGeneratorDriver.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.client;
-
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-
-public class SpatialQueryGeneratorDriver {
- public static void main(String[] args) throws Exception {
- SpatialQueryGeneratorConfig clientConfig = new SpatialQueryGeneratorConfig();
- CmdLineParser clp = new CmdLineParser(clientConfig);
- try {
- clp.parseArgument(args);
- } catch (CmdLineException e) {
- System.err.println(e.getMessage());
- clp.printUsage(System.err);
- System.exit(1);
- }
-
- SpatialQueryGenerator client = new SpatialQueryGenerator(clientConfig);
- client.start();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java
deleted file mode 100644
index 8621681..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorConfig.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.client;
-
-import org.kohsuke.args4j.Option;
-
-public class SyntheticDataGeneratorConfig {
-
- @Option(name = "-psf", aliases = "--point-source-file", usage = "The point source file")
- private String pointSourceFile;
-
- public String getPointSourceFile() {
- return pointSourceFile;
- }
-
- @Option(name = "-psi", aliases = "--point-sampling-interval", usage = "The point sampling interval from the point source file", required = true)
- private int pointSamplingInterval;
-
- public int getpointSamplingInterval() {
- return pointSamplingInterval;
- }
-
- @Option(name = "-pid", aliases = "--parition-id", usage = "The partition id in order to avoid key duplication", required = true)
- private int partitionId;
-
- public int getPartitionId() {
- return partitionId;
- }
-
- @Option(name = "-of", aliases = "--output-filepath", usage = "The output file path", required = true)
- private String outputFilePath;
-
- public String getOutputFilePath() {
- return outputFilePath;
- }
-
- @Option(name = "-rc", aliases = "--record-count", usage = "The record count to generate", required = true)
- private long recordCount;
-
- public long getRecordCount() {
- return recordCount;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java
deleted file mode 100644
index 25d56a0..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/client/SyntheticDataGeneratorForSpatialIndexEvaluation.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.client;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation;
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo;
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator;
-import org.apache.asterix.tools.external.data.GULongIDGenerator;
-import org.kohsuke.args4j.CmdLineException;
-import org.kohsuke.args4j.CmdLineParser;
-
-public class SyntheticDataGeneratorForSpatialIndexEvaluation {
- public static void main(String[] args) throws Exception {
- SyntheticDataGeneratorConfig config = new SyntheticDataGeneratorConfig();
- CmdLineParser clp = new CmdLineParser(config);
- try {
- clp.parseArgument(args);
- } catch (CmdLineException e) {
- System.err.println(e.getMessage());
- clp.printUsage(System.err);
- System.exit(1);
- }
-
- //prepare data generator
- GULongIDGenerator uidGenerator = new GULongIDGenerator(config.getPartitionId(), (byte) (0));
- String pointSourceFilePath = config.getPointSourceFile();
- int pointSampleInterval = config.getpointSamplingInterval();
- DataGeneratorForSpatialIndexEvaluation dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new InitializationInfo(), pointSourceFilePath,
- pointSampleInterval);
-
- //get record count to be generated
- long maxRecordCount = config.getRecordCount();
- long recordCount = 0;
-
- //prepare timer
- long startTS = System.currentTimeMillis();
-
- //prepare tweetIterator which acutally generates tweet
- TweetMessageIterator tweetIterator = dataGenerator.new TweetMessageIterator(0, uidGenerator);
-
- FileOutputStream fos = null;
- try {
- //prepare output file
- fos = openOutputFile(config.getOutputFilePath());
-
- while (recordCount < maxRecordCount) {
- //get a tweet and append newline at the end
- String tweet = tweetIterator.next().toString() + "\n";
- //write to file
- fos.write(tweet.getBytes());
-
- recordCount++;
- if (recordCount % 1000000 == 0) {
- System.out.println("... generated " + recordCount + " records");
- }
- }
- System.out.println("Done: generated " + recordCount + " records in "
- + ((System.currentTimeMillis() - startTS) / 1000) + " in seconds!");
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- try {
- closeOutputFile(fos);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- public static FileOutputStream openOutputFile(String filepath) throws IOException {
- File file = new File(filepath);
- if (file.exists()) {
- throw new IOException(filepath + "already exists");
- }
- file.createNewFile();
- return new FileOutputStream(file);
- }
-
- public static void closeOutputFile(FileOutputStream fos) throws IOException {
- fos.flush();
- fos.close();
- fos = null;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java
deleted file mode 100644
index 544fbf8..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/logging/ExperimentLogFormatter.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.logging;
-
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.text.MessageFormat;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.logging.Formatter;
-import java.util.logging.Level;
-import java.util.logging.LogRecord;
-
-public class ExperimentLogFormatter extends Formatter {
-
- // format string for printing the log record
- private static final String format = "{0} {1}\n{2}: {3}\n";
- private final Date dat = new Date();
- private final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
-
- /**
- * Format the given LogRecord.
- * <p>
- * The formatting can be customized by specifying the <a href="../Formatter.html#syntax">format string</a> in the <a href="#formatting"> {@code java.util.logging.SimpleFormatter.format}</a> property. The given {@code LogRecord} will be formatted as if by calling:
- *
- * <pre>
- * {@link String#format String.format}(format, date, source, logger, level, message, thrown);
- * </pre>
- *
- * where the arguments are:<br>
- * <ol>
- * <li>{@code format} - the {@link java.util.Formatter
- * java.util.Formatter} format string specified in the {@code java.util.logging.SimpleFormatter.format} property or the default format.</li>
- * <li>{@code date} - a {@link Date} object representing {@linkplain LogRecord#getMillis event time} of the log record.</li>
- * <li>{@code source} - a string representing the caller, if available; otherwise, the logger's name.</li>
- * <li>{@code logger} - the logger's name.</li>
- * <li>{@code level} - the {@linkplain Level#getLocalizedName
- * log level}.</li>
- * <li>{@code message} - the formatted log message returned from the {@link Formatter#formatMessage(LogRecord)} method. It uses {@link java.text.MessageFormat java.text} formatting and does not use the {@code java.util.Formatter format} argument.</li>
- * <li>{@code thrown} - a string representing the {@linkplain LogRecord#getThrown throwable} associated with the log record and its backtrace beginning with a newline character, if any; otherwise, an empty string.</li>
- * </ol>
- * <p>
- * Some example formats:<br>
- * <ul>
- * <li> {@code java.util.logging.SimpleFormatter.format="%4$s: %5$s [%1$tc]%n"}
- * <p>
- * This prints 1 line with the log level ({@code 4$}), the log message ({@code 5$}) and the timestamp ({@code 1$}) in a square bracket.
- *
- * <pre>
- * WARNING: warning message [Tue Mar 22 13:11:31 PDT 2011]
- * </pre>
- *
- * </li>
- * <li> {@code java.util.logging.SimpleFormatter.format="%1$tc %2$s%n%4$s: %5$s%6$s%n"}
- * <p>
- * This prints 2 lines where the first line includes the timestamp ({@code 1$}) and the source ({@code 2$}); the second line includes the log level ({@code 4$}) and the log message ({@code 5$}) followed with the throwable and its backtrace ({@code 6$}), if any:
- *
- * <pre>
- * Tue Mar 22 13:11:31 PDT 2011 MyClass fatal
- * SEVERE: several message with an exception
- * java.lang.IllegalArgumentException: invalid argument
- * at MyClass.mash(MyClass.java:9)
- * at MyClass.crunch(MyClass.java:6)
- * at MyClass.main(MyClass.java:3)
- * </pre>
- *
- * </li>
- * <li> {@code java.util.logging.SimpleFormatter.format="%1$tb %1$td, %1$tY %1$tl:%1$tM:%1$tS %1$Tp %2$s%n%4$s: %5$s%n"}
- * <p>
- * This prints 2 lines similar to the example above with a different date/time formatting and does not print the throwable and its backtrace:
- *
- * <pre>
- * Mar 22, 2011 1:11:31 PM MyClass fatal
- * SEVERE: several message with an exception
- * </pre>
- *
- * </li>
- * </ul>
- * <p>
- * This method can also be overridden in a subclass. It is recommended to use the {@link Formatter#formatMessage} convenience method to localize and format the message field.
- *
- * @param record
- * the log record to be formatted.
- * @return a formatted log record
- */
- public synchronized String format(LogRecord record) {
- dat.setTime(record.getMillis());
- String source;
- if (record.getSourceClassName() != null) {
- source = record.getSourceClassName();
- if (record.getSourceMethodName() != null) {
- source += " " + record.getSourceMethodName();
- }
- } else {
- source = record.getLoggerName();
- }
- String message = formatMessage(record);
- String throwable = "";
- if (record.getThrown() != null) {
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- pw.println();
- record.getThrown().printStackTrace(pw);
- pw.close();
- throwable = sw.toString();
- }
- return MessageFormat.format(format, sdf.format(dat), source, record.getLevel(),
- message, throwable);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java
deleted file mode 100644
index 13d2694..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractDynamicDataEvalReportBuilder.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.List;
-
-public abstract class AbstractDynamicDataEvalReportBuilder implements IDynamicDataEvalReportBuilder {
-
- protected final static String INSTANTANEOUS_INSERT_STRING = "[TimeToInsert100000]";
- protected final static int INSTANTAEOUS_INSERT_COUNT = 100000;
- protected final static int ROUND_COUNT = 721;
- protected final static int ROUND_INTERVAL = 5;
- protected final String expHomePath;
- protected final String expName;
- protected final String runLogFilePath;
- protected final String[] ncLogFilePaths;
- protected BufferedReader br = null;
- protected BufferedReader[] ncLogBrs = null;
- protected final int ncLogFileCount;
-
- protected final StringBuilder dataGenSb;
- protected final StringBuilder queryGenSb;
- protected final StringBuilder rsb;
-
- protected AbstractDynamicDataEvalReportBuilder(String expHomePath, String expName, String runLogFileName,
- boolean hasStatFile) {
- this.expHomePath = expHomePath;
- this.expName = expName;
- this.runLogFilePath = new String(expHomePath + runLogFileName);
- if (expName.contains("1A")) {
- ncLogFileCount = 1;
- } else if (expName.contains("1B")) {
- ncLogFileCount = 2;
- } else if (expName.contains("1C")) {
- ncLogFileCount = 4;
- } else /* if (expName.contains("1D") || other exps) */{
- ncLogFileCount = 8;
- }
- ncLogFilePaths = new String[ncLogFileCount];
- ncLogBrs = new BufferedReader[ncLogFileCount];
- for (int i = 0; i < ncLogFileCount; i++) {
- if (hasStatFile) {
- ncLogFilePaths[i] = new String(expHomePath + expName + File.separator + "node" + (i + 1)
- + File.separator + "logs" + File.separator + "a1_node" + (i + 1) + ".log");
- } else {
- ncLogFilePaths[i] = new String(expHomePath + expName + File.separator + "logs" + File.separator
- + "a1_node" + (i + 1) + ".log");
- }
- }
- dataGenSb = new StringBuilder();
- queryGenSb = new StringBuilder();
- rsb = new StringBuilder();
- }
-
- protected void openRunLog() throws IOException {
- br = new BufferedReader(new FileReader(runLogFilePath));
- for (int i = 0; i < ncLogFileCount; i++) {
- ncLogBrs[i] = new BufferedReader(new FileReader(ncLogFilePaths[i]));
- }
- }
-
- protected void closeRunLog() throws IOException {
- if (br != null) {
- br.close();
- }
- if (ncLogBrs != null) {
- for (int i = 0; i < ncLogFileCount; i++) {
- if (ncLogBrs[i] != null) {
- ncLogBrs[i].close();
- }
- }
- }
- }
-
- protected boolean moveToExperimentBegin() throws IOException {
- String line;
- while ((line = br.readLine()) != null) {
- if (line.contains("Running experiment: " + expName)) {
- return true;
- }
- }
- return false;
- }
-
- protected void renewStringBuilder() {
- dataGenSb.setLength(0);
- queryGenSb.setLength(0);
- rsb.setLength(0);
- }
-
- @Override
- public String getInstantaneousInsertPS(int nodeId, boolean useTimeForX) throws Exception {
- renewStringBuilder();
- openRunLog();
- try {
-
- if (!moveToExperimentBegin()) {
- //The experiment run log doesn't exist in this run log file
- return null;
- }
-
- int round = 0;
- while (round < ROUND_COUNT) {
- long IIPS = 0;
- String line;
- while ((line = ncLogBrs[nodeId].readLine()) != null) {
- if (line.contains("IPS")) {
- IIPS = ReportBuilderHelper.getLong(line, ", IIPS[", "]");
- break;
- }
- }
- round++;
- dataGenSb.append(round * ROUND_INTERVAL).append(",").append(IIPS).append("\n");
- }
-
- return dataGenSb.toString();
- } finally {
- closeRunLog();
- }
- }
-
- @Override
- public void getAllNodesAccumulatedInsertPS(int targetRound, List<Long> ipsList) throws Exception {
- renewStringBuilder();
- openRunLog();
- ipsList.clear();
- try {
-
- if (!moveToExperimentBegin()) {
- //The experiment run log doesn't exist in this run log file
- return;
- }
-
- int round = 0;
- while (round < targetRound) {
- long IPSPerRound = 0;
- for (int i = 0; i < ncLogFileCount; i++) {
- String line;
- while ((line = ncLogBrs[i].readLine()) != null) {
- if (line.contains("IPS")) {
- IPSPerRound += ReportBuilderHelper.getLong(line, ", IPS[", "]");
- break;
- }
- }
- }
- ipsList.add(IPSPerRound);
- round++;
- }
- return;
- } finally {
- closeRunLog();
- }
- }
-
- public String getInstantaneousDataGenPS(int genId, boolean useTimeForX) throws Exception {
- renewStringBuilder();
- openRunLog();
- try {
- if (!moveToExperimentBegin()) {
- //The experiment run log doesn't exist in this run log file
- return null;
- }
-
- String line;
- int dGenId;
- int count = 0;
- long timeToInsert = 0;
- long totalTimeToInsert = 0;
- while ((line = br.readLine()) != null) {
- if (line.contains(INSTANTANEOUS_INSERT_STRING)) {
- dGenId = ReportBuilderHelper.getInt(line, "DataGen[", "]");
- if (dGenId == genId) {
- count++;
- timeToInsert = ReportBuilderHelper.getLong(line, INSTANTANEOUS_INSERT_STRING, "in");
- totalTimeToInsert += timeToInsert;
- if (useTimeForX) {
- dataGenSb.append(totalTimeToInsert / 1000).append(",")
- .append(INSTANTAEOUS_INSERT_COUNT / ((double) (timeToInsert) / 1000)).append("\n");
- } else {
- dataGenSb.append(count).append(",")
- .append(INSTANTAEOUS_INSERT_COUNT / ((double) (timeToInsert) / 1000)).append("\n");
- }
- }
- }
- if (line.contains("Running")) {
- break;
- }
- }
- System.out.println("GenId[" + genId + "] " + totalTimeToInsert + ", " + (totalTimeToInsert / (1000 * 60)));
- return dataGenSb.toString();
- } finally {
- closeRunLog();
- }
- }
-
- public long getDataGenStartTimeStamp() throws Exception {
- openRunLog();
- try {
- String line;
- while ((line = br.readLine()) != null) {
- if (line.contains("Running experiment: " + expName)) {
- while ((line = br.readLine()) != null) {
- //2015-10-27 17:18:28,242 INFO [ParallelActionThread 6] transport.TransportImpl (TransportImpl.java:init(155)) - Client identity string: SSH-2.0-SSHJ_0_13_0
- if (line.contains("INFO [ParallelActionThread")) {
- //format1 = new SimpleDateFormat("MMM dd, yyyy hh:mm:ss aa");
- //format2 = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
- return ReportBuilderHelper.getTimeStampAsLong(line, format);
- }
- }
- }
- }
- return -1;
- } finally {
- closeRunLog();
- }
- }
-
- public String getIndexSize(String indexDirPath) throws Exception {
- /*
- * exmaple
- * /mnt/data/sdb/youngsk2/asterix/storage/experiments/Tweets_idx_dhbtreeLocation/device_id_0:
- total 211200
- -rw-r--r-- 1 youngsk2 grad 191234048 Jun 29 00:11 2015-06-29-00-09-59-023_2015-06-28-23-51-56-984_b
- -rw-r--r-- 1 youngsk2 grad 7864320 Jun 29 00:11 2015-06-29-00-09-59-023_2015-06-28-23-51-56-984_f
- -rw-r--r-- 1 youngsk2 grad 4194304 Jun 29 00:10 2015-06-29-00-10-26-997_2015-06-29-00-10-26-997_b
- -rw-r--r-- 1 youngsk2 grad 393216 Jun 29 00:10 2015-06-29-00-10-26-997_2015-06-29-00-10-26-997_f
- -rw-r--r-- 1 youngsk2 grad 5898240 Jun 29 00:11 2015-06-29-00-10-59-791_2015-06-29-00-10-59-791_b
- -rw-r--r-- 1 youngsk2 grad 393216 Jun 29 00:11 2015-06-29-00-10-59-791_2015-06-29-00-10-59-791_f
- -rw-r--r-- 1 youngsk2 grad 5898240 Jun 29 00:11 2015-06-29-00-11-30-486_2015-06-29-00-11-30-486_b
- -rw-r--r-- 1 youngsk2 grad 393216 Jun 29 00:11 2015-06-29-00-11-30-486_2015-06-29-00-11-30-486_f
-
- */
- renewStringBuilder();
- openRunLog();
- try {
- if (!moveToExperimentBegin()) {
- //The experiment run log doesn't exist in this run log file
- return null;
- }
-
- String line;
- String[] tokens;
- long diskSize = 0;
- while ((line = br.readLine()) != null) {
- if (line.contains(indexDirPath)) {
- br.readLine();//discard "total XXXX" line
- //read and sum file size
- while (!(line = br.readLine()).isEmpty()) {
- tokens = line.split("\\s+");;
- diskSize += Long.parseLong(tokens[4].trim());
- }
- }
- if (line.contains("Running")) {
- break;
- }
- }
- rsb.append((double) diskSize / (1024 * 1024 * 1024));
- return rsb.toString();
- } finally {
- closeRunLog();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java
deleted file mode 100644
index fdd1d92..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/AbstractStaticDataEvalReportBuilder.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-public abstract class AbstractStaticDataEvalReportBuilder implements IStaticDataEvalReportBuilder {
-
- protected final String expName;
- protected final String runLogFilePath;
-
- protected AbstractStaticDataEvalReportBuilder(String expName, String runLogFilePath) {
- this.expName = expName;
- this.runLogFilePath = runLogFilePath;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java
deleted file mode 100644
index 4d5e511..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IDynamicDataEvalReportBuilder.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-import java.util.List;
-
-public interface IDynamicDataEvalReportBuilder {
- public String getInstantaneousInsertPS(int genId, boolean useTimeForX) throws Exception;
-
- public String getOverallInsertPS(int minutes) throws Exception;
-
- public String getInstantaneousQueryPS() throws Exception;
-
- public String getQueryPS(int minutes) throws Exception;
-
- void getAllNodesAccumulatedInsertPS(int targetRound, List<Long> ipsList) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java
deleted file mode 100644
index 6c879f4..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/IStaticDataEvalReportBuilder.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-public interface IStaticDataEvalReportBuilder {
- public String getSelectQueryTime();
- public String getJoinQueryTime();
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java
deleted file mode 100644
index 5da4463..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/NCLogReportBuilder.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-public class NCLogReportBuilder {
-
- private String ncLogFilePath = "/Users/kisskys/workspace/asterix_experiment/run-log/measure-with-balloon/sie1-8dgen/log-1436511417368/SpatialIndexExperiment1ADhbtree/logs/a1_node1.log";
- private BufferedReader br;
- private String timeLine;
- private String msgLine;
-
- public NCLogReportBuilder(String filePath) {
- if (filePath != null) {
- this.ncLogFilePath = filePath;
- }
- }
-
- public String getFlushMergeEventAsGanttChartFormat(long testBeginTimeStamp) throws Exception {
- openNCLog();
- StringBuilder sb = new StringBuilder();
- long flushStartTimeStamp, flushFinishTimeStamp, mergeStartTimeStamp, mergeFinishTimeStamp;
- String indexName;
- SimpleDateFormat format = new SimpleDateFormat("MMM dd, yyyy hh:mm:ss aa");
- HashMap<String, Long> flushMap = new HashMap<String, Long>();
- HashMap<String, Long> mergeMap = new HashMap<String, Long>();
- long sTime, fTime;
- try {
- while ((timeLine = br.readLine()) != null) {
- if ((msgLine = br.readLine()) == null) {
- break;
- }
- while (!msgLine.contains("INFO:")) {
- timeLine = msgLine;
- msgLine = br.readLine();
- if (msgLine == null) {
- break;
- }
- }
- if (msgLine == null) {
- break;
- }
-
- //flush start
- if (msgLine.contains("Started a flush operation for index")) {
- flushStartTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format);
-
- //ignore flush op which happened before the data gen started.
- if (flushStartTimeStamp < testBeginTimeStamp) {
- continue;
- }
-
- indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]");
- flushMap.put(indexName, flushStartTimeStamp);
- }
-
- //flush finish
- if (msgLine.contains("Finished the flush operation for index")) {
- flushFinishTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format);
-
- //ignore flush op which happened before the data gen started.
- if (flushFinishTimeStamp < testBeginTimeStamp) {
- continue;
- }
-
- indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]");
-
- if (flushMap.containsKey(indexName)) {
- flushStartTimeStamp = flushMap.remove(indexName);
- sTime = (flushStartTimeStamp - testBeginTimeStamp) / 1000;
- fTime = (flushFinishTimeStamp - testBeginTimeStamp) / 1000;
- if (fTime == sTime) {
- ++fTime;
- }
- //only for sie1
- // if (fTime > 1200) {
- // fTime = 1200;
- // }
- sb.append("f-" + getPrintName(indexName)).append("\t").append(sTime).append("\t").append(fTime)
- .append("\t").append(indexName.contains("Tweets") ? "flushPidx" : "flushSidx")
- .append("\n");
- }
- }
-
- //merge start
- if (msgLine.contains("Started a merge operation for index")) {
- mergeStartTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format);
-
- //ignore flush op which happened before the data gen started.
- if (mergeStartTimeStamp < testBeginTimeStamp) {
- continue;
- }
-
- indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]");
- mergeMap.put(indexName, mergeStartTimeStamp);
- }
-
- //merge finish
- if (msgLine.contains("Finished the merge operation for index")) {
- mergeFinishTimeStamp = ReportBuilderHelper.getTimeStampAsLong(timeLine, format);
-
- //ignore flush op which happened before the data gen started.
- if (mergeFinishTimeStamp < testBeginTimeStamp) {
- continue;
- }
-
- indexName = ReportBuilderHelper.getString(msgLine, "experiments/Tweets_idx_", "/]");
-
- if (mergeMap.containsKey(indexName)) {
- mergeStartTimeStamp = mergeMap.remove(indexName);
- sTime = (mergeStartTimeStamp - testBeginTimeStamp) / 1000;
- fTime = (mergeFinishTimeStamp - testBeginTimeStamp) / 1000;
- if (fTime == sTime) {
- ++fTime;
- }
- //only for sie1
- // if (fTime > 1200) {
- // fTime = 1200;
- // }
- sb.append("m-" + getPrintName(indexName)).append("\t").append(sTime).append("\t").append(fTime)
- .append("\t").append(indexName.contains("Tweets") ? "mergePidx" : "mergeSidx")
- .append("\n");
- }
- }
- }
-
- Iterator<Entry<String, Long>> mergeMapIter = mergeMap.entrySet().iterator();
- Entry<String, Long> entry = null;
- while (mergeMapIter.hasNext()) {
- entry = mergeMapIter.next();
- sb.append("m-" + getPrintName(entry.getKey())).append("\t")
- .append((entry.getValue() - testBeginTimeStamp) / 1000).append("\t").append(60 * 20)
- .append("\t").append(entry.getKey().contains("Tweets") ? "mergePidx" : "mergeSidx")
- .append("\n");
- }
-
- Iterator<Entry<String, Long>> flushMapIter = mergeMap.entrySet().iterator();
- while (mergeMapIter.hasNext()) {
- entry = flushMapIter.next();
- sb.append("f-" + getPrintName(entry.getKey())).append("\t")
- .append((entry.getValue() - testBeginTimeStamp) / 1000).append("\t").append(60 * 20)
- .append("\t").append(entry.getKey().contains("Tweets") ? "flushPidx" : "flushSidx")
- .append("\n");
- }
-
- return sb.toString();
- } finally {
- closeNCLog();
- }
- }
-
- private String getPrintName(String indexName) {
- String name = null;
- if (indexName.contains("Tweets")) {
- if (indexName.contains("0")) {
- name = "pidx0";
- } else if (indexName.contains("1")) {
- name = "pidx1";
- } else if (indexName.contains("2")) {
- name = "pidx2";
- } else if (indexName.contains("3")) {
- name = "pidx3";
- }
- } else if (indexName.contains("Location")) {
- if (indexName.contains("0")) {
- name = "sidx0"; //ReportBuilderHelper.getString(indexName, "Location") + "0";
- } else if (indexName.contains("1")) {
- name = "sidx1"; //ReportBuilderHelper.getString(indexName, "Location") + "1";
- } else if (indexName.contains("2")) {
- name = "sidx2"; //ReportBuilderHelper.getString(indexName, "Location") + "2";
- } else if (indexName.contains("3")) {
- name = "sidx3"; //ReportBuilderHelper.getString(indexName, "Location") + "2";
- }
- }
- return name;
- }
-
- protected void openNCLog() throws IOException {
- br = new BufferedReader(new FileReader(ncLogFilePath));
- }
-
- protected void closeNCLog() throws IOException {
- if (br != null) {
- br.close();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java
deleted file mode 100644
index 24617bb..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/OperatorProfilerReportBuilder.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-public class OperatorProfilerReportBuilder {
-
- private static final int INDEX_BUILD_OP_COUNT = 1;
- private static final int PIDX_SCAN_OP_COUNT = 1;
- private static final int WARM_UP_SELECT_QUERY_COUNT = 500;
- private static final int SELECT_QUERY_COUNT = 5000;
- private static final int JOIN_QUERY_COUNT = 200;
- private static final int JOIN_RADIUS_TYPE_COUNT = 4;
- private static final int SELECT_RADIUS_TYPE_COUNT = 5;
- private static final int IDX_JOIN_RADIUS_SKIP = JOIN_RADIUS_TYPE_COUNT - 1;
- private static final int IDX_SELECT_RADIUS_SKIP = SELECT_RADIUS_TYPE_COUNT - 1;
- private static final int IDX_INITIAL_JOIN_SKIP = INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT
- + WARM_UP_SELECT_QUERY_COUNT + SELECT_QUERY_COUNT;
- private static final int IDX_INITIAL_SELECT_SKIP = INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT
- + WARM_UP_SELECT_QUERY_COUNT;
-
- private static final int HYRACK_JOB_ELAPSED_TIME_FIELD = 2;
- private static final int OP_ELAPSED_TIME_FIELD = 4;
- private static final int OP_TASK_ID_FIELD = 2;
- private static final int OP_NAME_FIELD = 1;
-
- private String executionTimeFilePath = null;
- private BufferedReader brExecutionTime;
- private String line;
- private int lineNum;
-
- public OperatorProfilerReportBuilder(String executionTimeFilePath) {
- this.executionTimeFilePath = executionTimeFilePath;
- }
-
- public String getIdxNumber(boolean isJoin, int radiusIdx) throws Exception {
- openExecutionTimeFile();
-
- StringBuilder sb = new StringBuilder();
- int initialSkip = (isJoin ? IDX_INITIAL_JOIN_SKIP : IDX_INITIAL_SELECT_SKIP) + radiusIdx;
- int radiusSkip = isJoin ? IDX_JOIN_RADIUS_SKIP : IDX_SELECT_RADIUS_SKIP;
- BufferedReader br = brExecutionTime;
- int queryCount = isJoin ? JOIN_QUERY_COUNT / JOIN_RADIUS_TYPE_COUNT : SELECT_QUERY_COUNT
- / SELECT_RADIUS_TYPE_COUNT;
- lineNum = 0;
- JobStat jobStat = new JobStat();
-
- try {
-
- //initial skip
- int jobCount = 0;
- while ((line = br.readLine()) != null) {
- lineNum++;
- if (line.contains("TOTAL_HYRACKS_JOB")) {
- jobCount++;
- if (jobCount > initialSkip) {
- break;
- }
- }
- }
-
- //Reaching Here, line variable contains the first job to be counted
- for (int j = 0; j < queryCount; j++) {
-
- analyzeOperatorExecutionTime(jobStat, br);
-
- //radius skip
- jobCount = 0;
- while ((line = br.readLine()) != null) {
- lineNum++;
- if (line.contains("TOTAL_HYRACKS_JOB")) {
- jobCount++;
- if (jobCount > radiusSkip) {
- break;
- }
- }
- }
- }
-
- //System.out.println("lineNum: " + lineNum);
- sb.append("TOTAL_HYRACKS_JOB," + (((double) jobStat.getHyracksJobTimeSum()) / jobStat.getHyracksJobCount())
- + "," + jobStat.getHyracksJobTimeSum() + "," + jobStat.getHyracksJobCount() + "\n");
- sb.append(jobStat.getOperatorsElapsedTimeAsString());
- return sb.toString();
- } finally {
- closeExecutionTimeFile();
- }
- }
-
- private void analyzeOperatorExecutionTime(JobStat jobStat, BufferedReader br) throws IOException {
- //the line argument contains TOTAL_HYRACKS_JOB string. eg.:
- //2015-11-04 19:13:08,003 TOTAL_HYRACKS_JOB a1_node1_JID:3_26202768 TOTAL_HYRACKS_JOB1446660788003 1066 1.066 1066 1.066
- String tokens[] = line.split("\t");
-
- if (Long.parseLong(tokens[HYRACK_JOB_ELAPSED_TIME_FIELD]) > 10000) {
- System.out.println("[" + lineNum + "] " + line);
- }
-
- jobStat.addHyracksJobTime(Long.parseLong(tokens[HYRACK_JOB_ELAPSED_TIME_FIELD]));
-
- while ((line = br.readLine()) != null) {
- lineNum++;
-
- if (line.isEmpty()) {
- break;
- }
-
- tokens = line.split("\t");
- if (line.contains("DISTRIBUTE_RESULT")) {
- jobStat.addDistributeResultTime(Long.parseLong(tokens[OP_ELAPSED_TIME_FIELD]));
- continue;
- }
- if (line.contains("EMPTY_TUPLE_SOURCE")) {
- continue;
- }
-
- if (line.contains("TXN_JOB_COMMIT")) {
- continue;
- }
-
- jobStat.updateOperatorTime(tokens[OP_TASK_ID_FIELD], tokens[OP_NAME_FIELD],
- Long.parseLong(tokens[OP_ELAPSED_TIME_FIELD]));
- }
-
- jobStat.updateTaskForAvgWithSlowestTask();
- }
-
- protected void openExecutionTimeFile() throws IOException {
- brExecutionTime = new BufferedReader(new FileReader(executionTimeFilePath));
- }
-
- protected void closeExecutionTimeFile() throws IOException {
- if (brExecutionTime != null) {
- brExecutionTime.close();
- }
- }
-
- class JobStat {
- private long hyracksJobElapsedTimeSum;
- private int hyracksJobCount;
- private long distributeResultTimeSum;
- private Task taskForAvg;
- private HashMap<String, Task> taskId2TaskMap;
-
- public JobStat() {
- hyracksJobElapsedTimeSum = 0;
- hyracksJobCount = 0;
- distributeResultTimeSum = 0;
- taskForAvg = new Task("TaskForAvg");
- taskId2TaskMap = new HashMap<String, Task>();
- }
-
- public void reset() {
- hyracksJobElapsedTimeSum = 0;
- hyracksJobCount = 0;
- distributeResultTimeSum = 0;
- taskForAvg.reset();;
- taskId2TaskMap.clear();
- }
-
- public void addHyracksJobTime(long elapsedTime) {
- hyracksJobElapsedTimeSum += elapsedTime;
- hyracksJobCount++;
- }
-
- public void addDistributeResultTime(long elapsedTime) {
- distributeResultTimeSum += elapsedTime;
- }
-
- public long getDistributeResultTime() {
- return distributeResultTimeSum;
- }
-
- public long getHyracksJobTimeSum() {
- return hyracksJobElapsedTimeSum;
- }
-
- public int getHyracksJobCount() {
- return hyracksJobCount;
- }
-
- public void updateOperatorTime(String taskId, String operatorName, long elapsedTime) {
- Task task = taskId2TaskMap.get(taskId);
- if (task == null) {
- task = new Task(taskId);
- taskId2TaskMap.put(new String(taskId), task);
- }
- task.updateOperatorTime(operatorName, elapsedTime);
- }
-
- public void updateTaskForAvgWithSlowestTask() {
- Iterator<Entry<String, Task>> taskIter = taskId2TaskMap.entrySet().iterator();
- Task slowestTask = null;
- Task curTask;
-
- //get the slowest task
- while (taskIter.hasNext()) {
- curTask = taskIter.next().getValue();
- if (slowestTask == null) {
- slowestTask = curTask;
- } else {
- if (slowestTask.getElapsedTime() < curTask.getElapsedTime()) {
- slowestTask = curTask;
- }
- }
- }
-
- //update the TaskForAvg with the slowest one
- HashMap<String, SumCount> operator2SumCountMap = slowestTask.getOperator2SumCountMap();
- Iterator<Entry<String, SumCount>> operatorIter = operator2SumCountMap.entrySet().iterator();
- while (operatorIter.hasNext()) {
- Entry<String, SumCount> entry = operatorIter.next();
- SumCount sc = entry.getValue();
- taskForAvg.updateOperatorTime(entry.getKey(), sc.sum);
- }
- taskId2TaskMap.clear();
- }
-
- public String getOperatorsElapsedTimeAsString() {
- return "SUM_OF_OPERATORS," + (((double) taskForAvg.getElapsedTime()) / hyracksJobCount) + ","
- + taskForAvg.getElapsedTime() + "," + hyracksJobCount + "\n"
- + taskForAvg.getOperatorsElapsedTimeAsString() + "DISTRIBUTE_RESULT,"
- + (((double) distributeResultTimeSum) / hyracksJobCount) + "," + distributeResultTimeSum + ","
- + hyracksJobCount + "\n";
- }
- }
-
- class Task {
- private String taskId;
- private long elapsedTime;
- private HashMap<String, SumCount> operator2SumCountMap;
-
- public Task(String taskId) {
- this.taskId = new String(taskId);
- elapsedTime = 0;
- operator2SumCountMap = new HashMap<String, SumCount>();
- }
-
- @Override
- public int hashCode() {
- return taskId.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
- if (!(o instanceof Task)) {
- return false;
- }
- return ((Task) o).taskId == taskId;
- }
-
- public long getElapsedTime() {
- return elapsedTime;
- }
-
- public void updateOperatorTime(String operatorName, long elapsedTime) {
- SumCount sc = operator2SumCountMap.get(operatorName);
- if (sc == null) {
- sc = new SumCount();
- sc.sum = 0;
- sc.count = 0;
- operator2SumCountMap.put(new String(operatorName), sc);
- }
- sc.sum += elapsedTime;
- sc.count++;
- this.elapsedTime += elapsedTime;
- }
-
- public void reset() {
- elapsedTime = 0;
- operator2SumCountMap.clear();
- }
-
- public String getOperatorsElapsedTimeAsString() {
- StringBuilder sb = new StringBuilder();
- Iterator<Entry<String, SumCount>> iter = operator2SumCountMap.entrySet().iterator();
- while (iter.hasNext()) {
- Entry<String, SumCount> entry = iter.next();
- SumCount sc = entry.getValue();
- sb.append(entry.getKey()).append(",").append(((double) sc.sum) / sc.count).append(",").append(sc.sum)
- .append(",").append(sc.count).append("\n");
- }
- return sb.toString();
- }
-
- public HashMap<String, SumCount> getOperator2SumCountMap() {
- return operator2SumCountMap;
- }
- }
-
- class SumCount {
- public long sum;
- public int count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/0e21afa7/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java b/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java
deleted file mode 100644
index d5eb539..0000000
--- a/asterixdb/asterix-experiments/src/main/java/org/apache/asterix/experiment/report/ProfilerReportBuilder.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.experiment.report;
-
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-
-public class ProfilerReportBuilder {
-
- private static final int INDEX_BUILD_OP_COUNT = 1;
- private static final int PIDX_SCAN_OP_COUNT = 1;
- private static final int WARM_UP_SELECT_QUERY_COUNT = 500;
- private static final int SELECT_QUERY_COUNT = 5000;
- private static final int JOIN_QUERY_COUNT = 200;
- private static final int PARTITION_COUNT = 4;
-
- private static final int SELECT_QUERY_INVOLVED_INDEX_COUNT = 2;
- private static final int JOIN_QUERY_INVOLVED_INDEX_COUNT = 3;
- private static final int JOIN_RADIUS_TYPE_COUNT = 4;
- private static final int SELECT_RADIUS_TYPE_COUNT = 5;
- private static final int IDX_JOIN_RADIUS_SKIP = JOIN_RADIUS_TYPE_COUNT * JOIN_QUERY_INVOLVED_INDEX_COUNT
- * PARTITION_COUNT - PARTITION_COUNT;
- private static final int IDX_SELECT_RADIUS_SKIP = SELECT_RADIUS_TYPE_COUNT * SELECT_QUERY_INVOLVED_INDEX_COUNT
- * PARTITION_COUNT - PARTITION_COUNT;
- private static final int IDX_INITIAL_JOIN_SKIP = (INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT + ((WARM_UP_SELECT_QUERY_COUNT + SELECT_QUERY_COUNT) * SELECT_QUERY_INVOLVED_INDEX_COUNT))
- * PARTITION_COUNT;
- private static final int IDX_INITIAL_SELECT_SKIP = (INDEX_BUILD_OP_COUNT + PIDX_SCAN_OP_COUNT + (WARM_UP_SELECT_QUERY_COUNT * SELECT_QUERY_INVOLVED_INDEX_COUNT))
- * PARTITION_COUNT;
- private static final int FP_JOIN_RADIUS_SKIP = JOIN_RADIUS_TYPE_COUNT * PARTITION_COUNT - PARTITION_COUNT;
- private static final int FP_SELECT_RADIUS_SKIP = SELECT_RADIUS_TYPE_COUNT * PARTITION_COUNT - PARTITION_COUNT;
- private static final int FP_INITIAL_JOIN_SKIP = (PIDX_SCAN_OP_COUNT + WARM_UP_SELECT_QUERY_COUNT + SELECT_QUERY_COUNT)
- * PARTITION_COUNT;
- private static final int FP_INITIAL_SELECT_SKIP = (PIDX_SCAN_OP_COUNT + WARM_UP_SELECT_QUERY_COUNT)
- * PARTITION_COUNT;
-
- private String indexSearchTimeFilePath = null;
- private String falsePositiveFilePath = null;
- private String cacheMissFilePath = null;
- private BufferedReader brIndexSearchTime;
- private BufferedReader brFalsePositive;
- private BufferedReader brCacheMiss;
- private String line;
-
- public ProfilerReportBuilder(String indexSearchTimeFilePath, String falsePositiveFilePath, String cacheMissFilePath) {
- this.indexSearchTimeFilePath = indexSearchTimeFilePath;
- this.falsePositiveFilePath = falsePositiveFilePath;
- this.cacheMissFilePath = cacheMissFilePath;
- }
-
- public String getIdxNumber(boolean getSearchTime, boolean isJoin, int radiusIdx, int indexIdx) throws Exception {
- if (getSearchTime) {
- openIndexSearchTimeFile();
- } else {
- openCacheMissFile();
- }
-
- StringBuilder sb = new StringBuilder();
- int involvedIndexCount = isJoin ? JOIN_QUERY_INVOLVED_INDEX_COUNT : SELECT_QUERY_INVOLVED_INDEX_COUNT;
- int initialSkip = (isJoin ? IDX_INITIAL_JOIN_SKIP : IDX_INITIAL_SELECT_SKIP) + radiusIdx * involvedIndexCount
- * PARTITION_COUNT + indexIdx * PARTITION_COUNT;
- int radiusSkip = isJoin ? IDX_JOIN_RADIUS_SKIP : IDX_SELECT_RADIUS_SKIP;
- long measuredValue = 0;
- BufferedReader br = getSearchTime ? brIndexSearchTime : brCacheMiss;
- int lineNum = 0;
- int queryCount = isJoin ? JOIN_QUERY_COUNT / JOIN_RADIUS_TYPE_COUNT : SELECT_QUERY_COUNT
- / SELECT_RADIUS_TYPE_COUNT;
- try {
-
- //initial skip
- for (int i = 0; i < initialSkip; i++) {
- br.readLine();
- ++lineNum;
- }
-
- for (int j = 0; j < queryCount; j++) {
- //get target index numbers
- for (int i = 0; i < PARTITION_COUNT; i++) {
- line = br.readLine();
- measuredValue += Long.parseLong(line);
- ++lineNum;
- }
-
- //radius skip
- for (int i = 0; i < radiusSkip; i++) {
- br.readLine();
- ++lineNum;
- }
- }
-
- //System.out.println("lineNum: " + lineNum);
- sb.append((double) measuredValue / (PARTITION_COUNT * queryCount));
- return sb.toString();
- } finally {
- if (getSearchTime) {
- closeIndexSearchTimeFile();
- } else {
- closeCacheMissFile();
- }
- }
- }
-
- public String getFalsePositives(boolean isJoin, int radiusIdx) throws Exception {
- openFalsePositiveFile();
-
- StringBuilder sb = new StringBuilder();
- int initialSkip = (isJoin ? FP_INITIAL_JOIN_SKIP : FP_INITIAL_SELECT_SKIP) + radiusIdx * PARTITION_COUNT;
- int radiusSkip = isJoin ? FP_JOIN_RADIUS_SKIP : FP_SELECT_RADIUS_SKIP;
- long falsePositives = 0;
- BufferedReader br = brFalsePositive;
- int lineNum = 0;
- int queryCount = isJoin ? JOIN_QUERY_COUNT / JOIN_RADIUS_TYPE_COUNT : SELECT_QUERY_COUNT
- / SELECT_RADIUS_TYPE_COUNT;
- try {
-
- //initial skip
- for (int i = 0; i < initialSkip; i++) {
- br.readLine();
- ++lineNum;
- }
-
- for (int j = 0; j < queryCount; j++) {
- //get target index numbers
- for (int i = 0; i < PARTITION_COUNT; i++) {
- line = br.readLine();
- falsePositives += Long.parseLong(line);
- ++lineNum;
- }
-
- //radius skip
- for (int i = 0; i < radiusSkip; i++) {
- br.readLine();
- ++lineNum;
- }
- }
-
- //System.out.println("lineNum: " + lineNum);
- sb.append((double) falsePositives / (PARTITION_COUNT * queryCount));
- return sb.toString();
- } finally {
- closeFalsePositiveFile();
- }
- }
-
- protected void openIndexSearchTimeFile() throws IOException {
- brIndexSearchTime = new BufferedReader(new FileReader(indexSearchTimeFilePath));
- }
-
- protected void closeIndexSearchTimeFile() throws IOException {
- if (brIndexSearchTime != null) {
- brIndexSearchTime.close();
- }
- }
-
- protected void openFalsePositiveFile() throws IOException {
- brFalsePositive = new BufferedReader(new FileReader(falsePositiveFilePath));
- }
-
- protected void closeFalsePositiveFile() throws IOException {
- if (brFalsePositive != null) {
- brFalsePositive.close();
- }
- }
-
- protected void openCacheMissFile() throws IOException {
- brCacheMiss = new BufferedReader(new FileReader(cacheMissFilePath));
- }
-
- protected void closeCacheMissFile() throws IOException {
- if (brCacheMiss != null) {
- brCacheMiss.close();
- }
- }
-
-}