You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/06/03 23:50:37 UTC
[1/2] hive git commit: HIVE-9814 : LLAP: JMX web-service end points
for monitoring & metrics (Gopal V, Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/llap cb24a7678 -> f032e5488
HIVE-9814 : LLAP: JMX web-service end points for monitoring & metrics (Gopal V, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/54127078
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/54127078
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/54127078
Branch: refs/heads/llap
Commit: 54127078f3a87b9f8dc9529467a04206be0cc398
Parents: cb24a76
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jun 3 14:25:38 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jun 3 14:25:38 2015 -0700
----------------------------------------------------------------------
.../llap/configuration/LlapConfiguration.java | 6 +++
llap-server/pom.xml | 11 ++++
.../hive/llap/daemon/impl/LlapDaemon.java | 6 +++
.../impl/LlapDaemonProtocolServerImpl.java | 2 +-
.../registry/impl/LlapRegistryService.java | 18 -------
.../registry/impl/LlapYarnRegistryImpl.java | 24 +++++++++
.../llap/daemon/services/impl/LlapWebApp.java | 12 +++++
.../daemon/services/impl/LlapWebServices.java | 53 ++++++++++++++++++++
.../src/main/resources/webapps/llap/.keep | 0
9 files changed, 113 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
index 194deee..b57ae80 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/configuration/LlapConfiguration.java
@@ -133,4 +133,10 @@ public class LlapConfiguration extends Configuration {
public static final String LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MILLIS =
LLAP_PREFIX + "task.communicator.connection.sleep-between-retries-millis";
public static final long LLAP_TASK_COMMUNICATOR_CONNECTION_SLEEP_BETWEEN_RETRIES_MILLIS_DEFAULT = 2000l;
+
+ public static final String LLAP_DAEMON_SERVICE_PORT = LLAP_DAEMON_PREFIX + "service.port";
+ public static final int LLAP_DAEMON_SERVICE_PORT_DEFAULT = 15002;
+
+ public static final String LLAP_DAEMON_SERVICE_SSL = LLAP_DAEMON_PREFIX + "service.ssl";
+ public static final boolean LLAP_DAEMON_SERVICE_SSL_DEFAULT = false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index 4fcd705..19f41fe 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -235,6 +235,17 @@
<build>
<sourceDirectory>${basedir}/src/java</sourceDirectory>
<testSourceDirectory>${basedir}/src/test</testSourceDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <excludes>
+ <exclude>*.py</exclude>
+ <exclude>*.pyc</exclude>
+ <exclude>llap-daemon-log4j.properties</exclude>
+ </excludes>
+ <filtering>false</filtering>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index d55b7cf..067e213 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryComp
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SourceStateUpdatedRequestProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto;
import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.TerminateFragmentRequestProto;
+import org.apache.hadoop.hive.llap.daemon.services.impl.LlapWebServices;
import org.apache.hadoop.hive.llap.io.api.LlapIoProxy;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorMetrics;
import org.apache.hadoop.hive.llap.metrics.LlapMetricsSystem;
@@ -54,6 +55,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap
private final LlapDaemonProtocolServerImpl server;
private final ContainerRunnerImpl containerRunner;
private final LlapRegistryService registry;
+ private final LlapWebServices webServices;
private final AtomicLong numSubmissions = new AtomicLong(0);
private JvmPauseMonitor pauseMonitor;
private final ObjectName llapDaemonInfoBean;
@@ -153,6 +155,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap
metrics);
this.registry = new LlapRegistryService(true);
+ this.webServices = new LlapWebServices();
}
private void printAsciiArt() {
@@ -174,6 +177,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap
server.init(conf);
containerRunner.init(conf);
registry.init(conf);
+ webServices.init(conf);
LlapIoProxy.setDaemon(true);
LlapIoProxy.initializeLlapIo(conf);
}
@@ -185,6 +189,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap
containerRunner.start();
registry.start();
registry.registerWorker();
+ webServices.start();
}
public void serviceStop() throws Exception {
@@ -194,6 +199,7 @@ public class LlapDaemon extends AbstractService implements ContainerRunner, Llap
server.stop();
registry.unregisterWorker();
registry.stop();
+ webServices.stop();
ShuffleHandler.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
index 8cb9715..cf3cc78 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemonProtocolServerImpl.java
@@ -68,7 +68,7 @@ public class LlapDaemonProtocolServerImpl extends AbstractService
}
@Override
- public SubmitWorkResponseProto submitWork(RpcController controller,
+ public SubmitWorkResponseProto submitWork (RpcController controller,
SubmitWorkRequestProto request) throws
ServiceException {
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
index 9f5702d..f727e09 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapRegistryService.java
@@ -14,31 +14,13 @@
package org.apache.hadoop.hive.llap.daemon.registry.impl;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.Map;
-import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
import org.apache.hadoop.hive.llap.daemon.registry.ServiceInstanceSet;
import org.apache.hadoop.hive.llap.daemon.registry.ServiceRegistry;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal;
-import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.log4j.Logger;
-import org.apache.zookeeper.CreateMode;
-
-import com.google.common.base.Preconditions;
public class LlapRegistryService extends AbstractService {
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
index ace080e..cb1b1d0 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/registry/impl/LlapYarnRegistryImpl.java
@@ -16,6 +16,9 @@ package org.apache.hadoop.hive.llap.daemon.registry.impl;
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.HashSet;
@@ -46,6 +49,7 @@ import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.log4j.Logger;
+import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.zookeeper.CreateMode;
import com.google.common.base.Preconditions;
@@ -117,6 +121,25 @@ public class LlapYarnRegistryImpl implements ServiceRegistry {
shufflePort);
}
+ public Endpoint getServicesEndpoint() {
+ final int servicePort =
+ conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_PORT,
+ LlapConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT);
+ final boolean isSSL =
+ conf.getBoolean(LlapConfiguration.LLAP_DAEMON_SERVICE_SSL,
+ LlapConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT);
+ final String scheme = isSSL ? "https" : "http";
+ final URL serviceURL;
+ try {
+ serviceURL = new URL(scheme, hostname, servicePort, "");
+ return RegistryTypeUtils.webEndpoint("services", serviceURL.toURI());
+ } catch (MalformedURLException e) {
+ throw new TezUncheckedException(e);
+ } catch (URISyntaxException e) {
+ throw new TezUncheckedException("llap service URI for " + hostname + " is invalid", e);
+ }
+ }
+
private final String getPath() {
return this.path;
}
@@ -127,6 +150,7 @@ public class LlapYarnRegistryImpl implements ServiceRegistry {
ServiceRecord srv = new ServiceRecord();
srv.addInternalEndpoint(getRpcEndpoint());
srv.addInternalEndpoint(getShuffleEndpoint());
+ srv.addExternalEndpoint(getServicesEndpoint());
for (Map.Entry<String, String> kv : this.conf) {
if (kv.getKey().startsWith(LlapConfiguration.LLAP_DAEMON_PREFIX)
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
new file mode 100644
index 0000000..cb57c9a
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebApp.java
@@ -0,0 +1,12 @@
+package org.apache.hadoop.hive.llap.daemon.services.impl;
+
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
+
+public class LlapWebApp extends WebApp {
+
+ @Override
+ public void setup() {
+ // JMX / config are defaults
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
new file mode 100644
index 0000000..2275719
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
@@ -0,0 +1,53 @@
+package org.apache.hadoop.hive.llap.daemon.services.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.llap.configuration.LlapConfiguration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.WebApp;
+import org.apache.hadoop.yarn.webapp.WebApps;
+import org.apache.log4j.Logger;
+
+public class LlapWebServices extends AbstractService {
+
+ private static final Logger LOG = Logger.getLogger(LlapWebServices.class);
+
+ private int port;
+ private boolean ssl;
+ private Configuration conf;
+ private WebApp webApp;
+ private LlapWebApp webAppInstance;
+
+ public LlapWebServices() {
+ super("LlapWebServices");
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+
+ this.conf = new Configuration(conf);
+ this.conf.addResource(YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
+
+ this.port = conf.getInt(LlapConfiguration.LLAP_DAEMON_SERVICE_PORT,
+ LlapConfiguration.LLAP_DAEMON_SERVICE_PORT_DEFAULT);
+ this.ssl = conf.getBoolean(LlapConfiguration.LLAP_DAEMON_SERVICE_SSL,
+ LlapConfiguration.LLAP_DAEMON_SERVICE_SSL_DEFAULT);
+
+ this.webAppInstance = new LlapWebApp();
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ String bindAddress = "0.0.0.0";
+ this.webApp =
+ WebApps.$for("llap").at(bindAddress).at(port).with(getConfig())
+ /* TODO: security negotiation here */
+ .start();
+ }
+
+ public void serviceStop() throws Exception {
+ if (this.webApp != null) {
+ this.webApp.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/54127078/llap-server/src/main/resources/webapps/llap/.keep
----------------------------------------------------------------------
diff --git a/llap-server/src/main/resources/webapps/llap/.keep b/llap-server/src/main/resources/webapps/llap/.keep
new file mode 100644
index 0000000..e69de29
[2/2] hive git commit: HIVE-10913 : LLAP: cache QF counters have a
wrong value for consumer time (Sergey Shelukhin)
Posted by se...@apache.org.
HIVE-10913 : LLAP: cache QF counters have a wrong value for consumer time (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f032e548
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f032e548
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f032e548
Branch: refs/heads/llap
Commit: f032e5488cc50f91c00b9c06ecc49b21146eb591
Parents: 5412707
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Jun 3 14:50:21 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Jun 3 14:50:21 2015 -0700
----------------------------------------------------------------------
.../hive/llap/counters/QueryFragmentCounters.java | 15 ++++++++++-----
.../hive/llap/io/api/impl/LlapInputFormat.java | 14 ++++++++++----
.../hive/llap/io/encoded/OrcEncodedDataReader.java | 2 ++
3 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f032e548/llap-server/src/java/org/apache/hadoop/hive/llap/counters/QueryFragmentCounters.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/counters/QueryFragmentCounters.java b/llap-server/src/java/org/apache/hadoop/hive/llap/counters/QueryFragmentCounters.java
index 7658b03..d7825a7 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/counters/QueryFragmentCounters.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/counters/QueryFragmentCounters.java
@@ -48,11 +48,14 @@ public class QueryFragmentCounters implements LowLevelCacheCounters {
}
public static enum Desc {
- TABLE
+ MACHINE,
+ TABLE,
+ FILE,
+ STRIPES
}
private final AtomicLongArray fixedCounters;
- private final String[] descs;
+ private final Object[] descs;
public QueryFragmentCounters(Configuration conf) {
fixedCounters = new AtomicLongArray(Counter.values().length);
@@ -62,7 +65,7 @@ public class QueryFragmentCounters implements LowLevelCacheCounters {
setCounter(Counter.TOTAL_IO_TIME_US, -1);
setCounter(Counter.DECODE_TIME_US, -1);
setCounter(Counter.HDFS_TIME_US, -1);
- setCounter(Counter.HDFS_TIME_US, -1);
+ setCounter(Counter.CONSUMER_TIME_US, -1);
}
}
@@ -88,7 +91,7 @@ public class QueryFragmentCounters implements LowLevelCacheCounters {
fixedCounters.set(counter.ordinal(), value);
}
- public void setDesc(Desc key, String desc) {
+ public void setDesc(Desc key, Object desc) {
descs[key.ordinal()] = desc;
}
@@ -122,7 +125,9 @@ public class QueryFragmentCounters implements LowLevelCacheCounters {
if (i != 0) {
sb.append(", ");
}
- sb.append(descs[i]);
+ if (descs[i] != null) {
+ sb.append(descs[i]);
+ }
}
sb.append("]: [ ");
for (int i = 0; i < fixedCounters.length(); ++i) {
http://git-wip-us.apache.org/repos/asf/hive/blob/f032e548/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
index a4f69da..e90dbbc 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.hive.common.util.HiveStringUtils;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
@@ -59,6 +60,7 @@ public class LlapInputFormat
private final InputFormat sourceInputFormat;
private final ColumnVectorProducer cvp;
private final ListeningExecutorService executor;
+ private final String hostName;
@SuppressWarnings("rawtypes")
LlapInputFormat(InputFormat sourceInputFormat, ColumnVectorProducer cvp,
@@ -69,6 +71,7 @@ public class LlapInputFormat
this.executor = executor;
this.cvp = cvp;
this.sourceInputFormat = sourceInputFormat;
+ this.hostName = HiveStringUtils.getHostname();
}
@Override
@@ -84,7 +87,7 @@ public class LlapInputFormat
try {
List<Integer> includedCols = ColumnProjectionUtils.isReadAllColumns(job)
? null : ColumnProjectionUtils.getReadColumnIDs(job);
- return new LlapRecordReader(job, fileSplit, includedCols);
+ return new LlapRecordReader(job, fileSplit, includedCols, hostName);
} catch (Exception ex) {
throw new IOException(ex);
}
@@ -114,12 +117,14 @@ public class LlapInputFormat
private final QueryFragmentCounters counters;
private long firstReturnTime;
- public LlapRecordReader(JobConf job, FileSplit split, List<Integer> includedCols) {
+ public LlapRecordReader(
+ JobConf job, FileSplit split, List<Integer> includedCols, String hostName) {
this.split = split;
this.columnIds = includedCols;
this.sarg = SearchArgumentFactory.createFromConf(job);
this.columnNames = ColumnProjectionUtils.getReadColumnNames(job);
this.counters = new QueryFragmentCounters(job);
+ this.counters.setDesc(QueryFragmentCounters.Desc.MACHINE, hostName);
try {
rbCtx = new VectorizedRowBatchCtx();
rbCtx.init(job, split);
@@ -136,6 +141,7 @@ public class LlapInputFormat
throw new AssertionError("next called after close");
}
// Add partition cols if necessary (see VectorizedOrcInputFormat for details).
+ boolean wasFirst = isFirst;
if (isFirst) {
try {
rbCtx.addPartitionColsToBatch(value);
@@ -153,7 +159,7 @@ public class LlapInputFormat
throw new IOException(e);
}
if (cvb == null) {
- if (isFirst) {
+ if (wasFirst) {
firstReturnTime = counters.startTimeCounter();
}
counters.incrTimeCounter(Counter.CONSUMER_TIME_US, firstReturnTime);
@@ -170,7 +176,7 @@ public class LlapInputFormat
}
value.selectedInUse = false;
value.size = cvb.size;
- if (isFirst) {
+ if (wasFirst) {
firstReturnTime = counters.startTimeCounter();
}
return true;
http://git-wip-us.apache.org/repos/asf/hive/blob/f032e548/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
index 6d2a38e..e107378 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java
@@ -143,6 +143,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
// Don't cache the filesystem object for now; Tez closes it and FS cache will fix all that
fs = split.getPath().getFileSystem(conf);
fileId = determineFileId(fs, split);
+ counters.setDesc(QueryFragmentCounters.Desc.FILE, fileId);
try {
fileMetadata = getOrReadFileMetadata();
@@ -165,6 +166,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void>
recordReaderTime(startTime);
return null; // No data to read.
}
+ counters.setDesc(QueryFragmentCounters.Desc.STRIPES, stripeIxFrom + "," + readState.length);
// 3. Apply SARG if needed, and otherwise determine what RGs to read.
int stride = fileMetadata.getRowIndexStride();