You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/08 20:43:22 UTC
[32/51] [partial] hive git commit: Revert "HIVE-14671 : merge master
into hive-14535 (Wei Zheng)"
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java b/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java
new file mode 100644
index 0000000..49d6d24
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFile.java
@@ -0,0 +1,273 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.beeline.qfile;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.util.Shell;
+import org.apache.hive.common.util.StreamPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * Class for representing a Query and the connected files. It provides accessors for the specific
+ * input and output files, and provides methods for filtering the output of the runs.
+ */
+public final class QFile {
+ private static final Logger LOG = LoggerFactory.getLogger(QFile.class.getName());
+
+ private String name;
+ private File inputFile;
+ private File rawOutputFile;
+ private File outputFile;
+ private File expcetedOutputFile;
+ private File logFile;
+ private File infraLogFile;
+ private static RegexFilterSet staticFilterSet = getStaticFilterSet();
+ private RegexFilterSet specificFilterSet;
+
+ private QFile() {}
+
+ public String getName() {
+ return name;
+ }
+
+ public File getInputFile() {
+ return inputFile;
+ }
+
+ public File getRawOutputFile() {
+ return rawOutputFile;
+ }
+
+ public File getOutputFile() {
+ return outputFile;
+ }
+
+ public File getExpectedOutputFile() {
+ return expcetedOutputFile;
+ }
+
+ public File getLogFile() {
+ return logFile;
+ }
+
+ public File getInfraLogFile() {
+ return infraLogFile;
+ }
+
+ public void filterOutput() throws IOException {
+ String rawOutput = FileUtils.readFileToString(rawOutputFile);
+ String filteredOutput = staticFilterSet.filter(specificFilterSet.filter(rawOutput));
+ FileUtils.writeStringToFile(outputFile, filteredOutput);
+ }
+
+ public boolean compareResults() throws IOException, InterruptedException {
+ if (!expcetedOutputFile.exists()) {
+ LOG.error("Expected results file does not exist: " + expcetedOutputFile);
+ return false;
+ }
+ return executeDiff();
+ }
+
+ public void overwriteResults() throws IOException {
+ if (expcetedOutputFile.exists()) {
+ FileUtils.forceDelete(expcetedOutputFile);
+ }
+ FileUtils.copyFile(outputFile, expcetedOutputFile);
+ }
+
+ private boolean executeDiff() throws IOException, InterruptedException {
+ List<String> diffCommandArgs = new ArrayList<String>();
+ diffCommandArgs.add("diff");
+
+ // Text file comparison
+ diffCommandArgs.add("-a");
+
+ if (Shell.WINDOWS) {
+ // Ignore changes in the amount of white space
+ diffCommandArgs.add("-b");
+
+ // Files created on Windows machines have different line endings
+ // than files created on Unix/Linux. Windows uses carriage return and line feed
+ // ("\r\n") as a line ending, whereas Unix uses just line feed ("\n").
+ // Also StringBuilder.toString(), Stream to String conversions adds extra
+ // spaces at the end of the line.
+ diffCommandArgs.add("--strip-trailing-cr"); // Strip trailing carriage return on input
+ diffCommandArgs.add("-B"); // Ignore changes whose lines are all blank
+ }
+
+ // Add files to compare to the arguments list
+ diffCommandArgs.add(getQuotedString(expcetedOutputFile));
+ diffCommandArgs.add(getQuotedString(outputFile));
+
+ System.out.println("Running: " + org.apache.commons.lang.StringUtils.join(diffCommandArgs,
+ ' '));
+ Process executor = Runtime.getRuntime().exec(diffCommandArgs.toArray(
+ new String[diffCommandArgs.size()]));
+
+ StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, System.err);
+ StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out);
+
+ outPrinter.start();
+ errPrinter.start();
+
+ int result = executor.waitFor();
+
+ outPrinter.join();
+ errPrinter.join();
+
+ executor.waitFor();
+
+ return (result == 0);
+ }
+
+ private static String getQuotedString(File file) {
+ return Shell.WINDOWS ? String.format("\"%s\"", file.getAbsolutePath()) : file.getAbsolutePath();
+ }
+
+ private static class RegexFilterSet {
+ private final Map<Pattern, String> regexFilters = new LinkedHashMap<Pattern, String>();
+
+ public RegexFilterSet addFilter(String regex, String replacement) {
+ regexFilters.put(Pattern.compile(regex), replacement);
+ return this;
+ }
+
+ public String filter(String input) {
+ for (Pattern pattern : regexFilters.keySet()) {
+ input = pattern.matcher(input).replaceAll(regexFilters.get(pattern));
+ }
+ return input;
+ }
+ }
+
+ // These are the filters which are common for every QTest.
+ // Check specificFilterSet for QTest specific ones.
+ private static RegexFilterSet getStaticFilterSet() {
+ // Extract the leading four digits from the unix time value.
+ // Use this as a prefix in order to increase the selectivity
+ // of the unix time stamp replacement regex.
+ String currentTimePrefix = Long.toString(System.currentTimeMillis()).substring(0, 4);
+
+ String userName = System.getProperty("user.name");
+
+ String timePattern = "(Mon|Tue|Wed|Thu|Fri|Sat|Sun) "
+ + "(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) "
+ + "\\d{2} \\d{2}:\\d{2}:\\d{2} \\w+ 20\\d{2}";
+ // Pattern to remove the timestamp and other infrastructural info from the out file
+ String logPattern = "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2},\\d*\\s+\\S+\\s+\\[" +
+ ".*\\]\\s+\\S+:\\s+";
+ String operatorPattern = "\"(CONDITION|COPY|DEPENDENCY_COLLECTION|DDL"
+ + "|EXPLAIN|FETCH|FIL|FS|FUNCTION|GBY|HASHTABLEDUMMY|HASTTABLESINK|JOIN"
+ + "|LATERALVIEWFORWARD|LIM|LVJ|MAP|MAPJOIN|MAPRED|MAPREDLOCAL|MOVE|OP|RS"
+ + "|SCR|SEL|STATS|TS|UDTF|UNION)_\\d+\"";
+
+ return new RegexFilterSet()
+ .addFilter(logPattern, "")
+ .addFilter("Getting log thread is interrupted, since query is done!\n", "")
+ .addFilter("going to print operations logs\n", "")
+ .addFilter("printed operations logs\n", "")
+ .addFilter("\\(queryId=[^\\)]*\\)", "queryId=(!!{queryId}!!)")
+ .addFilter("file:/\\w\\S+", "file:/!!ELIDED!!")
+ .addFilter("pfile:/\\w\\S+", "pfile:/!!ELIDED!!")
+ .addFilter("hdfs:/\\w\\S+", "hdfs:/!!ELIDED!!")
+ .addFilter("last_modified_by=\\w+", "last_modified_by=!!ELIDED!!")
+ .addFilter(timePattern, "!!TIMESTAMP!!")
+ .addFilter("(\\D)" + currentTimePrefix + "\\d{6}(\\D)", "$1!!UNIXTIME!!$2")
+ .addFilter("(\\D)" + currentTimePrefix + "\\d{9}(\\D)", "$1!!UNIXTIMEMILLIS!!$2")
+ .addFilter(userName, "!!{user.name}!!")
+ .addFilter(operatorPattern, "\"$1_!!ELIDED!!\"")
+ .addFilter("Time taken: [0-9\\.]* seconds", "Time taken: !!ELIDED!! seconds");
+ }
+
+ /**
+ * Builder to generate QFile objects. After initializing the builder it is possible the
+ * generate the next QFile object using it's name only.
+ */
+ public static class QFileBuilder {
+ private File queryDirectory;
+ private File logDirectory;
+ private File resultsDirectory;
+ private String scratchDirectoryString;
+ private String warehouseDirectoryString;
+ private File hiveRootDirectory;
+
+ public QFileBuilder() {
+ }
+
+ public QFileBuilder setQueryDirectory(File queryDirectory) {
+ this.queryDirectory = queryDirectory;
+ return this;
+ }
+
+ public QFileBuilder setLogDirectory(File logDirectory) {
+ this.logDirectory = logDirectory;
+ return this;
+ }
+
+ public QFileBuilder setResultsDirectory(File resultsDirectory) {
+ this.resultsDirectory = resultsDirectory;
+ return this;
+ }
+
+ public QFileBuilder setScratchDirectoryString(String scratchDirectoryString) {
+ this.scratchDirectoryString = scratchDirectoryString;
+ return this;
+ }
+
+ public QFileBuilder setWarehouseDirectoryString(String warehouseDirectoryString) {
+ this.warehouseDirectoryString = warehouseDirectoryString;
+ return this;
+ }
+
+ public QFileBuilder setHiveRootDirectory(File hiveRootDirectory) {
+ this.hiveRootDirectory = hiveRootDirectory;
+ return this;
+ }
+
+ public QFile getQFile(String name) throws IOException {
+ QFile result = new QFile();
+ result.name = name;
+ result.inputFile = new File(queryDirectory, name + ".q");
+ result.rawOutputFile = new File(logDirectory, name + ".q.out.raw");
+ result.outputFile = new File(logDirectory, name + ".q.out");
+ result.expcetedOutputFile = new File(resultsDirectory, name + ".q.out");
+ result.logFile = new File(logDirectory, name + ".q.beeline");
+ result.infraLogFile = new File(logDirectory, name + ".q.out.infra");
+ // These are the filters which are specific for the given QTest.
+ // Check staticFilterSet for common filters.
+ result.specificFilterSet = new RegexFilterSet()
+ .addFilter(scratchDirectoryString + "[\\w\\-/]+", "!!{hive.exec.scratchdir}!!")
+ .addFilter(warehouseDirectoryString, "!!{hive.metastore.warehouse.dir}!!")
+ .addFilter(resultsDirectory.getAbsolutePath(), "!!{expectedDirectory}!!")
+ .addFilter(logDirectory.getAbsolutePath(), "!!{outputDirectory}!!")
+ .addFilter(queryDirectory.getAbsolutePath(), "!!{qFileDirectory}!!")
+ .addFilter(hiveRootDirectory.getAbsolutePath(), "!!{hive.root}!!");
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java b/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java
new file mode 100644
index 0000000..b6eac89
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/beeline/qfile/QFileBeeLineClient.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.beeline.qfile;
+
+import org.apache.hive.beeline.BeeLine;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * QFile test client using BeeLine. It can be used to submit a list of command strings, or a QFile.
+ */
+public class QFileBeeLineClient implements AutoCloseable {
+ private BeeLine beeLine;
+ private PrintStream beelineOutputStream;
+ private File logFile;
+
+ protected QFileBeeLineClient(String jdbcUrl, String jdbcDriver, String username, String password,
+ File log) throws IOException {
+ logFile = log;
+ beeLine = new BeeLine();
+ beelineOutputStream = new PrintStream(logFile, "UTF-8");
+ beeLine.setOutputStream(beelineOutputStream);
+ beeLine.setErrorStream(beelineOutputStream);
+ beeLine.runCommands(
+ new String[] {
+ "!set verbose true",
+ "!set shownestederrs true",
+ "!set showwarnings true",
+ "!set showelapsedtime false",
+ "!set maxwidth -1",
+ "!connect " + jdbcUrl + " " + username + " " + password + " " + jdbcDriver
+ });
+ }
+
+ public boolean execute(String[] commands, File resultFile) {
+ boolean hasErrors = false;
+ beeLine.runCommands(
+ new String[] {
+ "!set outputformat csv",
+ "!record " + resultFile.getAbsolutePath()
+ });
+
+ if (commands.length != beeLine.runCommands(commands)) {
+ hasErrors = true;
+ }
+
+ beeLine.runCommands(new String[] {"!record"});
+ return !hasErrors;
+ }
+
+ private void beforeExecute(QFile qFile) {
+ assert(execute(
+ new String[] {
+ "USE default;",
+ "SHOW TABLES;",
+ "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;",
+ "CREATE DATABASE `" + qFile.getName() + "`;",
+ "USE `" + qFile.getName() + "`;"
+ },
+ qFile.getInfraLogFile()));
+ }
+
+ private void afterExecute(QFile qFile) {
+ assert(execute(
+ new String[] {
+ "USE default;",
+ "DROP DATABASE IF EXISTS `" + qFile.getName() + "` CASCADE;",
+ },
+ qFile.getInfraLogFile()));
+ }
+
+ public boolean execute(QFile qFile) {
+ beforeExecute(qFile);
+ boolean result = execute(
+ new String[] {
+ "!run " + qFile.getInputFile().getAbsolutePath()
+ },
+ qFile.getRawOutputFile());
+ afterExecute(qFile);
+ return result;
+ }
+
+ public void close() {
+ if (beeLine != null) {
+ beeLine.runCommands(new String[] {
+ "!quit"
+ });
+ }
+ if (beelineOutputStream != null) {
+ beelineOutputStream.close();
+ }
+ }
+
+ /**
+ * Builder to generated QFileBeeLineClient objects. The after initializing the builder, it can be
+ * used to create new clients without any parameters.
+ */
+ public static class QFileClientBuilder {
+ private String username;
+ private String password;
+ private String jdbcUrl;
+ private String jdbcDriver;
+
+ public QFileClientBuilder() {
+ }
+
+ public QFileClientBuilder setUsername(String username) {
+ this.username = username;
+ return this;
+ }
+
+ public QFileClientBuilder setPassword(String password) {
+ this.password = password;
+ return this;
+ }
+
+ public QFileClientBuilder setJdbcUrl(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
+ return this;
+ }
+
+ public QFileClientBuilder setJdbcDriver(String jdbcDriver) {
+ this.jdbcDriver = jdbcDriver;
+ return this;
+ }
+
+ public QFileBeeLineClient getClient(File logFile) throws IOException {
+ return new QFileBeeLineClient(jdbcUrl, jdbcDriver, username, password, logFile);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/itests/util/src/main/java/org/apache/hive/beeline/qfile/package-info.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hive/beeline/qfile/package-info.java b/itests/util/src/main/java/org/apache/hive/beeline/qfile/package-info.java
new file mode 100644
index 0000000..fcd50ec
--- /dev/null
+++ b/itests/util/src/main/java/org/apache/hive/beeline/qfile/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package for the BeeLine specific QTest file classes.
+ */
+package org.apache.hive.beeline.qfile;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/jdbc-handler/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc-handler/pom.xml b/jdbc-handler/pom.xml
index 6c6e1fa..364886a 100644
--- a/jdbc-handler/pom.xml
+++ b/jdbc-handler/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 1294a61..8adf67b 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -168,18 +168,6 @@
</includes>
</filter>
<filter>
- <artifact>org.apache.parquet:parquet-hadoop-bundle</artifact>
- <excludes>
- <exclude>shaded/parquet/org/codehaus/jackson/**</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>org.apache.logging.log4j:log4j-core</artifact>
- <excludes>
- <exclude>org/apache/logging/log4j/core/jackson/**</exclude>
- </excludes>
- </filter>
- <filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
@@ -192,7 +180,11 @@
<artifactSet>
<excludes>
<exclude>org.apache.commons:commons-compress</exclude>
- <exclude>org.apache.hadoop:*</exclude>
+ <exclude>org.apache.hadoop:hadoop-yarn*</exclude>
+ <exclude>org.apache.hadoop:hadoop-mapreduce*</exclude>
+ <exclude>org.apache.hadoop:hadoop-hdfs</exclude>
+ <exclude>org.apache.hadoop:hadoop-client</exclude>
+ <exclude>org.apache.hadoop:hadoop-annotations</exclude>
<exclude>org.apache.hive:hive-vector-code-gen</exclude>
<exclude>org.apache.ant:*</exclude>
<exclude>junit:*</exclude>
@@ -237,7 +229,6 @@
<exclude>com.thoughtworks.paranamer:*</exclude>
<exclude>com.twitter:*</exclude>
<exclude>com.zaxxer:*</exclude>
- <exclude>com.fasterxml.jackson.core:*</exclude>
<exclude>io.netty:*</exclude>
<exclude>javax.activation:*</exclude>
<exclude>javax.inject:*</exclude>
@@ -301,6 +292,13 @@
<shadedPattern>org.apache.hive.com.facebook</shadedPattern>
</relocation>
<relocation>
+ <pattern>org.apache.hadoop</pattern>
+ <shadedPattern>org.apache.hive.org.apache.hadoop</shadedPattern>
+ <excludes>
+ <exclude>org.apache.hadoop.security.*</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
<pattern>org.apache.zookeeper</pattern>
<shadedPattern>org.apache.hive.org.apache.zookeeper</shadedPattern>
</relocation>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index fb18adb..1695c5d 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -41,7 +41,6 @@ import org.apache.hive.service.rpc.thrift.TSessionHandle;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.HttpResponse;
import org.apache.http.client.CookieStore;
-import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.ServiceUnavailableRetryStrategy;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
@@ -387,9 +386,9 @@ public class HiveConnection implements java.sql.Connection {
* Add an interceptor to pass username/password in the header.
* In https mode, the entire information is encrypted
*/
- requestInterceptor =
- new HttpBasicAuthInterceptor(getUserName(), getPassword(), cookieStore, cookieName,
- useSsl, additionalHttpHeaders);
+ requestInterceptor = new HttpBasicAuthInterceptor(getUserName(), getPassword(),
+ cookieStore, cookieName, useSsl,
+ additionalHttpHeaders);
}
}
// Configure http client for cookie based authentication
@@ -422,23 +421,6 @@ public class HiveConnection implements java.sql.Connection {
} else {
httpClientBuilder = HttpClientBuilder.create();
}
- // In case the server's idletimeout is set to a lower value, it might close it's side of
- // connection. However we retry one more time on NoHttpResponseException
- httpClientBuilder.setRetryHandler(new HttpRequestRetryHandler() {
- @Override
- public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
- if (executionCount > 1) {
- LOG.info("Retry attempts to connect to server exceeded.");
- return false;
- }
- if (exception instanceof org.apache.http.NoHttpResponseException) {
- LOG.info("Could not connect to the server. Retrying one more time.");
- return true;
- }
- return false;
- }
- });
-
// Add the request interceptor to the client builder
httpClientBuilder.addInterceptorFirst(requestInterceptor);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index c385e2c..a0aea72 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -315,11 +315,9 @@ public class HiveStatement implements java.sql.Statement {
isExecuteStatementFailed = false;
} catch (SQLException eS) {
isExecuteStatementFailed = true;
- isLogBeingGenerated = false;
throw eS;
} catch (Exception ex) {
isExecuteStatementFailed = true;
- isLogBeingGenerated = false;
throw new SQLException(ex.toString(), "08S01", ex);
}
}
@@ -916,6 +914,10 @@ public class HiveStatement implements java.sql.Statement {
if (isQueryClosed) {
throw new ClosedOrCancelledStatementException("Method getQueryLog() failed. The " +
"statement has been closed or cancelled.");
+ }
+ if (isExecuteStatementFailed) {
+ throw new SQLException("Method getQueryLog() failed. Because the stmtHandle in " +
+ "HiveStatement is null and the statement execution might fail.");
} else {
return logs;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/jdbc/src/java/org/apache/hive/jdbc/logs/InPlaceUpdateStream.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/logs/InPlaceUpdateStream.java b/jdbc/src/java/org/apache/hive/jdbc/logs/InPlaceUpdateStream.java
index 90b7368..d4cd79c 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/logs/InPlaceUpdateStream.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/logs/InPlaceUpdateStream.java
@@ -1,20 +1,3 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package org.apache.hive.jdbc.logs;
import org.apache.hive.service.rpc.thrift.TProgressUpdateResp;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-client/pom.xml
----------------------------------------------------------------------
diff --git a/llap-client/pom.xml b/llap-client/pom.xml
index aa2cf32..3bacd2b 100644
--- a/llap-client/pom.xml
+++ b/llap-client/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
index 42129b7..e5ab601 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/io/api/LlapIo.java
@@ -25,5 +25,4 @@ import org.apache.hadoop.mapred.InputFormat;
public interface LlapIo<T> {
InputFormat<NullWritable, T> getInputFormat(InputFormat sourceInputFormat, Deserializer serde);
void close();
- String getMemoryInfo();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
index 76fc9c7..610c0a5 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapRegistryService.java
@@ -61,7 +61,7 @@ public class LlapRegistryService extends AbstractService {
if (hosts.startsWith("@")) {
// Caching instances only in case of the YARN registry. Each host based list will get it's own copy.
String appName = hosts.substring(1);
- String userName = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, currentUser());
+ String userName = HiveConf.getVar(conf, ConfVars.LLAP_ZK_REGISTRY_USER, RegistryUtils.currentUser());
String key = appName + "-" + userName;
registry = yarnRegistries.get(key);
if (registry == null || !registry.isInState(STATE.STARTED)) {
@@ -79,9 +79,6 @@ public class LlapRegistryService extends AbstractService {
return registry;
}
- public static String currentUser() {
- return RegistryUtils.currentUser();
- }
@Override
public void serviceInit(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-common/pom.xml
----------------------------------------------------------------------
diff --git a/llap-common/pom.xml b/llap-common/pom.xml
index bb40996..334fd75 100644
--- a/llap-common/pom.xml
+++ b/llap-common/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
deleted file mode 100644
index fa29b59..0000000
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.llap;
-
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.yarn.api.records.Resource;
-
-public enum LlapDaemonInfo {
- INSTANCE;
-
- private static final class LlapDaemonInfoHolder {
- public LlapDaemonInfoHolder(int numExecutors, long executorMemory, long cacheSize,
- boolean isDirectCache, boolean isLlapIo) {
- this.numExecutors = numExecutors;
- this.executorMemory = executorMemory;
- this.cacheSize = cacheSize;
- this.isDirectCache = isDirectCache;
- this.isLlapIo = isLlapIo;
- }
-
- final int numExecutors;
- final long executorMemory;
- final long cacheSize;
- final boolean isDirectCache;
- final boolean isLlapIo;
- }
-
- // add more variables as required
- private AtomicReference<LlapDaemonInfoHolder> dataRef =
- new AtomicReference<LlapDaemonInfoHolder>();
-
- public static void initialize(String appName, Configuration daemonConf) {
- int numExecutors = HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_NUM_EXECUTORS);
- long executorMemoryBytes =
- HiveConf.getIntVar(daemonConf, ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB) * 1024l * 1024l;
- long ioMemoryBytes = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
- boolean isDirectCache = HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_ALLOCATOR_DIRECT);
- boolean isLlapIo = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED, true);
- initialize(appName, numExecutors, executorMemoryBytes, ioMemoryBytes, isDirectCache, isLlapIo);
- }
-
- public static void initialize(String appName, int numExecutors, long executorMemoryBytes,
- long ioMemoryBytes, boolean isDirectCache, boolean isLlapIo) {
- INSTANCE.dataRef.set(new LlapDaemonInfoHolder(numExecutors, executorMemoryBytes, ioMemoryBytes,
- isDirectCache, isLlapIo));
- }
-
- public boolean isLlap() {
- return dataRef.get() != null;
- }
-
- public int getNumExecutors() {
- return dataRef.get().numExecutors;
- }
-
- public long getExecutorMemory() {
- return dataRef.get().executorMemory;
- }
-
- public long getMemoryPerExecutor() {
- final LlapDaemonInfoHolder data = dataRef.get();
- return (getExecutorMemory() - -(data.isDirectCache ? 0 : data.cacheSize)) / getNumExecutors();
- }
-
- public long getCacheSize() {
- return dataRef.get().cacheSize;
- }
-
- public boolean isDirectCache() {
- return dataRef.get().isDirectCache;
- }
-
- public boolean isLlapIo() {
- return dataRef.get().isLlapIo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-ext-client/pom.xml
----------------------------------------------------------------------
diff --git a/llap-ext-client/pom.xml b/llap-ext-client/pom.xml
index d9ea026..5ba0ec5 100644
--- a/llap-ext-client/pom.xml
+++ b/llap-ext-client/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/bin/runLlapDaemon.sh
----------------------------------------------------------------------
diff --git a/llap-server/bin/runLlapDaemon.sh b/llap-server/bin/runLlapDaemon.sh
index 82c2cc5..001e304 100755
--- a/llap-server/bin/runLlapDaemon.sh
+++ b/llap-server/bin/runLlapDaemon.sh
@@ -51,7 +51,7 @@ shift
JAVA=$JAVA_HOME/bin/java
LOG_LEVEL_DEFAULT="INFO"
LOGGER_DEFAULT="console"
-JAVA_OPTS_BASE="-server -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+PrintGCDetails -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=4 -XX:GCLogFileSize=100M -XX:+PrintGCDateStamps"
+JAVA_OPTS_BASE="-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+PrintGCDetails -verbose:gc -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=4 -XX:GCLogFileSize=100M -XX:+PrintGCDateStamps"
if [ ! -d "${LLAP_DAEMON_HOME}" ]; then
echo No LLAP_DAEMON_HOME set, or is not a directory.
@@ -107,7 +107,7 @@ elif [ "$COMMAND" = "run" ] ; then
CLASS='org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon'
fi
-JAVA_OPTS_BASE="${JAVA_OPTS_BASE} -Xloggc:${LLAP_DAEMON_LOG_DIR}/gc_$(date +%Y-%m-%d-%H).log"
+JAVA_OPTS_BASE="${JAVA_OPTS_BASE} -Xloggc:${LLAP_DAEMON_LOG_DIR}/gc.log"
LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} ${JAVA_OPTS_BASE}"
# Set the default GC option if none set
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/pom.xml
----------------------------------------------------------------------
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index b10f05f..630e243 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hive</groupId>
<artifactId>hive</artifactId>
- <version>3.0.0-SNAPSHOT</version>
+ <version>2.2.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
@@ -178,10 +178,6 @@
<version>${slider.version}</version>
<exclusions>
<exclusion>
- <groupId>asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
@@ -226,8 +222,8 @@
<artifactId>jettison</artifactId>
</exclusion>
<exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
</exclusion>
</exclusions>
</dependency>
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
index 6cf8dbb..ff6e7ce 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java
@@ -116,7 +116,7 @@ public class IncrementalObjectSizeEstimator {
addToProcessing(byType, stack, fieldObj, fieldClass);
}
}
- estimator.directSize = (int) JavaDataModel.alignUp(
+ estimator.directSize = JavaDataModel.alignUp(
estimator.directSize, memoryModel.memoryAlign());
}
}
@@ -454,7 +454,7 @@ public class IncrementalObjectSizeEstimator {
if (len != 0) {
int elementSize = getPrimitiveSize(e.field.getType().getComponentType());
arraySize += elementSize * len;
- arraySize = (int) JavaDataModel.alignUp(arraySize, memoryModel.memoryAlign());
+ arraySize = JavaDataModel.alignUp(arraySize, memoryModel.memoryAlign());
}
referencedSize += arraySize;
break;
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
index 302918a..8d7f0d3 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hive.llap.cache;
-import java.util.concurrent.atomic.AtomicLong;
-
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -45,14 +43,11 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl;
import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics;
-public final class BuddyAllocator
- implements EvictionAwareAllocator, BuddyAllocatorMXBean, LlapOomDebugDump {
+public final class BuddyAllocator implements EvictionAwareAllocator, BuddyAllocatorMXBean {
private final Arena[] arenas;
private final AtomicInteger allocatedArenas = new AtomicInteger(0);
private final MemoryManager memoryManager;
- private static final long MAX_DUMP_INTERVAL_NS = 300 * 1000000000L; // 5 minutes.
- private final AtomicLong lastLog = new AtomicLong(-1);
// Config settings
private final int minAllocLog2, maxAllocLog2, arenaSizeLog2, maxArenas;
@@ -124,14 +119,13 @@ public final class BuddyAllocator
} else {
cacheDir = null;
}
- long arenaSizeVal = (arenaCount == 0) ? MAX_ARENA_SIZE : maxSizeVal / arenaCount;
- // The math.min, and the fact that maxAllocation is an int, ensures we don't overflow.
+ int arenaSizeVal = (arenaCount == 0) ? MAX_ARENA_SIZE : (int)(maxSizeVal / arenaCount);
arenaSizeVal = Math.max(maxAllocation, Math.min(arenaSizeVal, MAX_ARENA_SIZE));
if (LlapIoImpl.LOG.isInfoEnabled()) {
- LlapIoImpl.LOG.info("Buddy allocator with " + (isDirect ? "direct" : "byte") + " buffers; "
- + (isMapped ? ("memory mapped off " + cacheDir.toString() + "; ") : "")
+ LlapIoImpl.LOG.info("Buddy allocator with " + (isDirect ? "direct" : "byte") + " buffers;"
+ + (isMapped ? (" memory mapped off " + cacheDir.toString() + "; ") : "")
+ "allocation sizes " + minAllocation + " - " + maxAllocation
- + ", arena size " + arenaSizeVal + ", total size " + maxSizeVal);
+ + ", arena size " + arenaSizeVal + ". total size " + maxSizeVal);
}
String minName = ConfVars.LLAP_ALLOCATOR_MIN_ALLOC.varname,
@@ -154,7 +148,7 @@ public final class BuddyAllocator
LlapIoImpl.LOG.warn("Rounding arena size to " + arenaSizeVal + " from " + oldArenaSize
+ " to be divisible by allocation size " + maxAllocation);
}
- arenaSize = (int)arenaSizeVal;
+ arenaSize = arenaSizeVal;
if ((maxSizeVal % arenaSize) > 0) {
long oldMaxSize = maxSizeVal;
maxSizeVal = (maxSizeVal / arenaSize) * arenaSize;
@@ -197,7 +191,8 @@ public final class BuddyAllocator
int allocLog2 = freeListIx + minAllocLog2;
int allocationSize = 1 << allocLog2;
// TODO: reserving the entire thing is not ideal before we alloc anything. Interleave?
- memoryManager.reserveMemory(dest.length << allocLog2);
+ memoryManager.reserveMemory(dest.length << allocLog2, true);
+
int destAllocIx = 0;
for (int i = 0; i < dest.length; ++i) {
if (dest[i] != null) continue;
@@ -246,106 +241,38 @@ public final class BuddyAllocator
// into some sort of queues that deallocate and split will examine), or having and "actor"
// allocator thread (or threads per arena).
// The 2nd one is probably much simpler and will allow us to get rid of a lot of sync code.
- // But for now we will just retry. We will evict more each time.
- long forceReserved = 0;
- int attempt = 0;
- try {
- while (true) {
- // Try to split bigger blocks. TODO: again, ideally we would tryLock at least once
- {
- int startArenaIx = (int)((threadId + attempt) % arenaCount), arenaIx = startArenaIx;
- do {
- int newDestIx = arenas[arenaIx].allocateWithSplit(
- arenaIx, freeListIx, dest, destAllocIx, allocationSize);
- if (newDestIx == dest.length) return;
- assert newDestIx != -1;
- destAllocIx = newDestIx;
- if ((++arenaIx) == arenaCount) {
- arenaIx = 0;
- }
- } while (arenaIx != startArenaIx);
- }
-
- if (attempt == 0) {
- // Try to allocate memory if we haven't allocated all the way to maxSize yet; very rare.
- for (int arenaIx = arenaCount; arenaIx < arenas.length; ++arenaIx) {
- destAllocIx = arenas[arenaIx].allocateWithExpand(
- arenaIx, freeListIx, dest, destAllocIx, allocationSize);
- if (destAllocIx == dest.length) return;
+ // But for now we will just retry 5 times 0_o
+ for (int attempt = 0; attempt < 5; ++attempt) {
+ // Try to split bigger blocks. TODO: again, ideally we would tryLock at least once
+ {
+ int startArenaIx = (int)((threadId + attempt) % arenaCount), arenaIx = startArenaIx;
+ do {
+ int newDestIx = arenas[arenaIx].allocateWithSplit(
+ arenaIx, freeListIx, dest, destAllocIx, allocationSize);
+ if (newDestIx == dest.length) return;
+ assert newDestIx != -1;
+ destAllocIx = newDestIx;
+ if ((++arenaIx) == arenaCount) {
+ arenaIx = 0;
}
- }
- int numberToForce = (dest.length - destAllocIx) * (attempt + 1);
- long newReserved = memoryManager.forceReservedMemory(allocationSize, numberToForce);
- forceReserved += newReserved;
- if (newReserved == 0) {
- // Cannot force-evict anything, give up.
- String msg = "Failed to allocate " + size + "; at " + destAllocIx + " out of "
- + dest.length + " (entire cache is fragmented and locked, or an internal issue)";
- logOomErrorMessage(msg);
- throw new AllocatorOutOfMemoryException(msg);
- }
- if (attempt == 0) {
- LlapIoImpl.LOG.warn("Failed to allocate despite reserved memory; will retry");
- }
- ++attempt;
- }
- } finally {
- if (attempt > 4) {
- LlapIoImpl.LOG.warn("Allocation of " + dest.length + " buffers of size " + size
- + " took " + attempt + " attempts to evict enough memory");
- }
- // After we succeed (or fail), release the force-evicted memory to memory manager. We have
- // previously reserved enough to allocate all we need, so we don't take our allocation out
- // of this - as per the comment above, we basically just wasted a bunch of cache (and CPU).
- if (forceReserved > 0) {
- memoryManager.releaseMemory(forceReserved);
+ } while (arenaIx != startArenaIx);
}
- }
- }
- private void logOomErrorMessage(String msg) {
- while (true) {
- long time = System.nanoTime();
- long lastTime = lastLog.get();
- // Magic value usage is invalid with nanoTime, so once in a 1000 years we may log extra.
- boolean shouldLog = (lastTime == -1 || (time - lastTime) > MAX_DUMP_INTERVAL_NS);
- if (shouldLog && !lastLog.compareAndSet(lastTime, time)) {
- continue;
- }
- if (shouldLog) {
- LlapIoImpl.LOG.error(msg + debugDumpForOom());
- } else {
- LlapIoImpl.LOG.error(msg);
- }
- return;
- }
- }
-
- /**
- * Arbitrarily, we start getting the state from Allocator. Allocator calls MM which calls
- * the policies that call the eviction dispatcher that calls the caches. See init - these all
- * are connected in a cycle, so we need to make sure the who-calls-whom order is definite.
- */
- @Override
- public void debugDumpShort(StringBuilder sb) {
- memoryManager.debugDumpShort(sb);
- sb.append("\nAllocator state:");
- int unallocCount = 0, fullCount = 0;
- long totalFree = 0;
- for (Arena arena : arenas) {
- Integer result = arena.debugDumpShort(sb);
- if (result == null) {
- ++unallocCount;
- } else if (result == 0) {
- ++fullCount;
- } else {
- totalFree += result;
+ if (attempt == 0) {
+ // Try to allocate memory if we haven't allocated all the way to maxSize yet; very rare.
+ for (int arenaIx = arenaCount; arenaIx < arenas.length; ++arenaIx) {
+ destAllocIx = arenas[arenaIx].allocateWithExpand(
+ arenaIx, freeListIx, dest, destAllocIx, allocationSize);
+ if (destAllocIx == dest.length) return;
+ }
}
+ memoryManager.forceReservedMemory(allocationSize, dest.length - destAllocIx);
+ LlapIoImpl.LOG.warn("Failed to allocate despite reserved memory; will retry " + attempt);
}
- sb.append("\nTotal available and allocated: ").append(totalFree).append(
- "; unallocated arenas: ").append(unallocCount).append(
- "; full arenas ").append(fullCount);
- sb.append("\n");
+ String msg = "Failed to allocate " + size + "; at " + destAllocIx + " out of " + dest.length;
+ LlapIoImpl.LOG.error(msg + "\nALLOCATOR STATE:\n" + debugDump()
+ + "\nPARENT STATE:\n" + memoryManager.debugDumpForOom());
+ throw new AllocatorOutOfMemoryException(msg);
}
@Override
@@ -372,7 +299,7 @@ public final class BuddyAllocator
return isDirect;
}
- public String debugDumpForOomInternal() {
+ public String debugDump() {
StringBuilder result = new StringBuilder(
"NOTE: with multiple threads the dump is not guaranteed to be consistent");
for (Arena arena : arenas) {
@@ -469,36 +396,6 @@ public final class BuddyAllocator
}
}
- public Integer debugDumpShort(StringBuilder result) {
- if (data == null) {
- return null;
- }
- int allocSize = minAllocation;
- int total = 0;
- for (int i = 0; i < freeLists.length; ++i, allocSize <<= 1) {
- FreeList freeList = freeLists[i];
- freeList.lock.lock();
- try {
- int nextHeaderIx = freeList.listHead;
- int count = 0;
- while (nextHeaderIx >= 0) {
- ++count;
- nextHeaderIx = getNextFreeListItem(offsetFromHeaderIndex(nextHeaderIx));
- }
- if (count > 0) {
- if (total == 0) {
- result.append("\nArena with free list lengths by size: ");
- }
- total += (allocSize * count);
- result.append(allocSize).append(" => ").append(count).append(", ");
- }
- } finally {
- freeList.lock.unlock();
- }
- }
- return total;
- }
-
public void debugDump(StringBuilder result) {
result.append("\nArena: ");
if (data == null) {
@@ -780,10 +677,4 @@ public final class BuddyAllocator
public MemoryBuffer createUnallocated() {
return new LlapDataBuffer();
}
-
- @Override
- public String debugDumpForOom() {
- return "\nALLOCATOR STATE:\n" + debugDumpForOomInternal()
- + "\nPARENT STATE:\n" + memoryManager.debugDumpForOom();
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
index c73f1a1..a6b0abd 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/EvictionDispatcher.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata;
/**
* Eviction dispatcher - uses double dispatch to route eviction notifications to correct caches.
*/
-public final class EvictionDispatcher implements EvictionListener, LlapOomDebugDump {
+public final class EvictionDispatcher implements EvictionListener {
private final LowLevelCache dataCache;
private final SerDeLowLevelCacheImpl serdeCache;
private final OrcMetadataCache metadataCache;
@@ -65,27 +65,4 @@ public final class EvictionDispatcher implements EvictionListener, LlapOomDebugD
public void notifyEvicted(OrcFileEstimateErrors buffer) {
metadataCache.notifyEvicted(buffer);
}
-
- @Override
- public String debugDumpForOom() {
- StringBuilder sb = new StringBuilder(dataCache.debugDumpForOom());
- if (serdeCache != null) {
- sb.append(serdeCache.debugDumpForOom());
- }
- if (metadataCache != null) {
- sb.append(metadataCache.debugDumpForOom());
- }
- return sb.toString();
- }
-
- @Override
- public void debugDumpShort(StringBuilder sb) {
- dataCache.debugDumpShort(sb);
- if (serdeCache != null) {
- serdeCache.debugDumpShort(sb);
- }
- if (metadataCache != null) {
- metadataCache.debugDumpShort(sb);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapOomDebugDump.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapOomDebugDump.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapOomDebugDump.java
index e861a7e..30bf5a9 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapOomDebugDump.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LlapOomDebugDump.java
@@ -20,5 +20,4 @@ package org.apache.hadoop.hive.llap.cache;
public interface LlapOomDebugDump {
String debugDumpForOom();
- void debugDumpShort(StringBuilder sb);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
index c5d0c84..19c589a 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hive.common.io.DataCache.BooleanRef;
import org.apache.hadoop.hive.common.io.DataCache.DiskRangeListFactory;
import org.apache.hadoop.hive.common.io.encoded.MemoryBuffer;
-public interface LowLevelCache extends LlapOomDebugDump {
+public interface LowLevelCache {
public enum Priority {
NORMAL,
HIGH
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
index 23796f6..4dc1c23 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java
@@ -466,43 +466,4 @@ public class LowLevelCacheImpl implements LowLevelCache, BufferUsageManager, Lla
}
return sb.toString();
}
-
- @Override
- public void debugDumpShort(StringBuilder sb) {
- sb.append("\nORC cache state ");
- int allLocked = 0, allUnlocked = 0, allEvicted = 0;
- for (Map.Entry<Object, FileCache<ConcurrentSkipListMap<Long, LlapDataBuffer>>> e :
- cache.entrySet()) {
- if (!e.getValue().incRef()) continue;
- try {
- int fileLocked = 0, fileUnlocked = 0, fileEvicted = 0;
- if (e.getValue().getCache().isEmpty()) continue;
- for (Map.Entry<Long, LlapDataBuffer> e2 : e.getValue().getCache().entrySet()) {
- int newRc = e2.getValue().incRef();
- if (newRc < 0) {
- ++fileEvicted;
- continue;
- }
- try {
- if (newRc > 1) { // We hold one refcount.
- ++fileLocked;
- } else {
- ++fileUnlocked;
- }
- } finally {
- e2.getValue().decRef();
- }
- }
- allLocked += fileLocked;
- allUnlocked += fileUnlocked;
- allEvicted += fileEvicted;
- sb.append("\n file " + e.getKey() + ": " + fileLocked + " locked, "
- + fileUnlocked + " unlocked, " + fileEvicted + " evicted");
- } finally {
- e.getValue().decRef();
- }
- }
- sb.append("\nORC cache summary: " + allLocked + " locked, "
- + allUnlocked + " unlocked, " + allEvicted + " evicted");
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
index 2132574..88bfa8b 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheMemoryManager.java
@@ -38,28 +38,25 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
private long maxSize;
public LowLevelCacheMemoryManager(
+ Configuration conf, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
+ this(HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), evictor, metrics);
+ }
+
+ @VisibleForTesting
+ public LowLevelCacheMemoryManager(
long maxSize, LowLevelCachePolicy evictor, LlapDaemonCacheMetrics metrics) {
this.maxSize = maxSize;
this.evictor = evictor;
this.usedMemory = new AtomicLong(0);
this.metrics = metrics;
+ metrics.setCacheCapacityTotal(maxSize);
if (LlapIoImpl.LOG.isInfoEnabled()) {
LlapIoImpl.LOG.info("Memory manager initialized with max size {} and" +
" {} ability to evict blocks", maxSize, ((evictor == null) ? "no " : ""));
}
}
-
@Override
- public void reserveMemory(final long memoryToReserve) {
- boolean result = reserveMemory(memoryToReserve, true);
- if (result) return;
- // Can only happen if there's no evictor, or if thread is interrupted.
- throw new RuntimeException("Cannot reserve memory"
- + (Thread.currentThread().isInterrupted() ? "; thread interrupted" : ""));
- }
-
- @VisibleForTesting
public boolean reserveMemory(final long memoryToReserve, boolean waitForEviction) {
// TODO: if this cannot evict enough, it will spin infinitely. Terminate at some point?
int badCallCount = 0;
@@ -111,19 +108,19 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
usedMem = usedMemory.get();
}
}
- if (!result) {
- releaseMemory(reservedTotalMetric);
- reservedTotalMetric = 0;
- }
metrics.incrCacheCapacityUsed(reservedTotalMetric - evictedTotalMetric);
return result;
}
@Override
- public long forceReservedMemory(int allocationSize, int count) {
- if (evictor == null) return 0;
- return evictor.tryEvictContiguousData(allocationSize, count);
+ public void forceReservedMemory(int allocationSize, int count) {
+ if (evictor == null) return;
+ while (count > 0) {
+ int evictedCount = evictor.tryEvictContiguousData(allocationSize, count);
+ if (evictedCount == 0) return;
+ count -= evictedCount;
+ }
}
@Override
@@ -139,13 +136,7 @@ public class LowLevelCacheMemoryManager implements MemoryManager {
@Override
public String debugDumpForOom() {
if (evictor == null) return null;
- return "\ncache state\n" + evictor.debugDumpForOom();
- }
-
- @Override
- public void debugDumpShort(StringBuilder sb) {
- if (evictor == null) return;
- evictor.debugDumpShort(sb);
+ return "cache state\n" + evictor.debugDumpForOom();
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
index fd9d942..bb1d1b0 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCachePolicy.java
@@ -28,5 +28,5 @@ public interface LowLevelCachePolicy extends LlapOomDebugDump {
void setEvictionListener(EvictionListener listener);
void setParentDebugDumper(LlapOomDebugDump dumper);
/** TODO: temporary method until we have a better allocator */
- long tryEvictContiguousData(int allocationSize, int count);
+ int tryEvictContiguousData(int allocationSize, int count);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
index 761fd00..430a5f8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelFifoCachePolicy.java
@@ -34,7 +34,7 @@ public class LowLevelFifoCachePolicy implements LowLevelCachePolicy {
private EvictionListener evictionListener;
private LlapOomDebugDump parentDebugDump;
- public LowLevelFifoCachePolicy() {
+ public LowLevelFifoCachePolicy(Configuration conf) {
LlapIoImpl.LOG.info("FIFO cache policy");
buffers = new LinkedList<LlapCacheableBuffer>();
}
@@ -116,26 +116,10 @@ public class LowLevelFifoCachePolicy implements LowLevelCachePolicy {
}
@Override
- public void debugDumpShort(StringBuilder sb) {
- sb.append("\nFIFO eviction list: ");
- lock.lock();
- try {
- sb.append(buffers.size()).append(" elements)");
- } finally {
- lock.unlock();
- }
- if (parentDebugDump != null) {
- parentDebugDump.debugDumpShort(sb);
- }
- }
-
- @Override
- public long tryEvictContiguousData(int allocationSize, int count) {
+ public int tryEvictContiguousData(int allocationSize, int count) {
long evicted = evictInternal(allocationSize * count, allocationSize);
- int remainingCount = count - (int)(evicted / allocationSize);
- if (remainingCount > 0) {
- evicted += evictInternal(allocationSize * remainingCount, -1);
- }
- return evicted;
+ // This makes granularity assumptions.
+ assert evicted % allocationSize == 0;
+ return (int)(evicted / allocationSize);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
index 3973c8a..4cd2c18 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelLrfuCachePolicy.java
@@ -69,6 +69,12 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
private EvictionListener evictionListener;
private LlapOomDebugDump parentDebugDump;
+ public LowLevelLrfuCachePolicy(Configuration conf) {
+ this((int)HiveConf.getSizeVar(conf, ConfVars.LLAP_ALLOCATOR_MIN_ALLOC),
+ HiveConf.getSizeVar(conf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE), conf);
+ }
+
+ @VisibleForTesting
public LowLevelLrfuCachePolicy(int minBufferSize, long maxSize, Configuration conf) {
lambda = HiveConf.getFloatVar(conf, HiveConf.ConfVars.LLAP_LRFU_LAMBDA);
int maxBuffers = (int)Math.ceil((maxSize * 1.0) / minBufferSize);
@@ -204,14 +210,13 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
}
@Override
- public long tryEvictContiguousData(int allocationSize, int count) {
+ public int tryEvictContiguousData(int allocationSize, int count) {
int evicted = evictDataFromList(allocationSize, count);
- if (count <= evicted) return evicted * allocationSize;
- evicted += evictDataFromHeap(timer.get(), count - evicted, allocationSize);
- long evictedBytes = evicted * allocationSize;
- if (count <= evicted) return evictedBytes;
- evictedBytes += evictSomeBlocks(allocationSize * (count - evicted));
- return evictedBytes;
+ count -= evicted;
+ if (count > 0) {
+ evicted += evictDataFromHeap(timer.get(), count, allocationSize);
+ }
+ return evicted;
}
private long evictFromList(long memoryToReserve) {
@@ -568,26 +573,4 @@ public class LowLevelLrfuCachePolicy implements LowLevelCachePolicy {
}
return result;
}
-
- @Override
- public void debugDumpShort(StringBuilder sb) {
- sb.append("\nLRFU eviction list: ");
- LlapCacheableBuffer listHeadLocal = listHead, listTailLocal = listTail;
- if (listHeadLocal == null) {
- sb.append("0 items");
- } else {
- LlapCacheableBuffer listItem = listHeadLocal;
- int c = 0;
- while (listItem != null) {
- ++c;
- if (listItem == listTailLocal) break;
- listItem = listItem.next;
- }
- sb.append(c + " items");
- }
- sb.append("\nLRFU eviction heap: " + heapSize + " items");
- if (parentDebugDump != null) {
- parentDebugDump.debugDumpShort(sb);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
index 0f4d3c0..ca41825 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/MemoryManager.java
@@ -19,9 +19,9 @@
package org.apache.hadoop.hive.llap.cache;
public interface MemoryManager extends LlapOomDebugDump {
+ boolean reserveMemory(long memoryToReserve, boolean waitForEviction);
void releaseMemory(long memUsage);
void updateMaxSize(long maxSize);
/** TODO: temporary method until we get a better allocator. */
- long forceReservedMemory(int allocationSize, int count);
- void reserveMemory(long memoryToReserve);
+ void forceReservedMemory(int allocationSize, int count);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SerDeLowLevelCacheImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SerDeLowLevelCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SerDeLowLevelCacheImpl.java
index cd5bc9b..4809398 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SerDeLowLevelCacheImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SerDeLowLevelCacheImpl.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -43,7 +44,7 @@ import org.apache.orc.OrcProto.ColumnEncoding;
import com.google.common.base.Function;
-public class SerDeLowLevelCacheImpl implements LlapOomDebugDump {
+public class SerDeLowLevelCacheImpl implements BufferUsageManager, LlapOomDebugDump {
private static final int DEFAULT_CLEANUP_INTERVAL = 600;
private final Allocator allocator;
private final AtomicInteger newEvictions = new AtomicInteger(0);
@@ -616,6 +617,18 @@ public class SerDeLowLevelCacheImpl implements LlapOomDebugDump {
}
}
+ @Override
+ public void decRefBuffer(MemoryBuffer buffer) {
+ unlockBuffer((LlapDataBuffer)buffer, true);
+ }
+
+ @Override
+ public void decRefBuffers(List<MemoryBuffer> cacheBuffers) {
+ for (MemoryBuffer b : cacheBuffers) {
+ unlockBuffer((LlapDataBuffer)b, true);
+ }
+ }
+
private void unlockBuffer(LlapDataBuffer buffer, boolean handleLastDecRef) {
boolean isLastDecref = (buffer.decRef() == 0);
if (handleLastDecRef && isLastDecref) {
@@ -691,6 +704,18 @@ public class SerDeLowLevelCacheImpl implements LlapOomDebugDump {
}
@Override
+ public boolean incRefBuffer(MemoryBuffer buffer) {
+ // notifyReused implies that buffer is already locked; it's also called once for new
+ // buffers that are not cached yet. Don't notify cache policy.
+ return lockBuffer(((LlapDataBuffer)buffer), false);
+ }
+
+ @Override
+ public Allocator getAllocator() {
+ return allocator;
+ }
+
+ @Override
public String debugDumpForOom() {
StringBuilder sb = new StringBuilder("File cache state ");
for (Map.Entry<Object, FileCache<FileData>> e : cache.entrySet()) {
@@ -706,55 +731,4 @@ public class SerDeLowLevelCacheImpl implements LlapOomDebugDump {
}
return sb.toString();
}
-
-
- @Override
- public void debugDumpShort(StringBuilder sb) {
- sb.append("\nSerDe cache state ");
- int allLocked = 0, allUnlocked = 0, allEvicted = 0;
- for (Map.Entry<Object, FileCache<FileData>> e : cache.entrySet()) {
- if (!e.getValue().incRef()) continue;
- try {
- FileData fd = e.getValue().getCache();
- int fileLocked = 0, fileUnlocked = 0, fileEvicted = 0;
- sb.append(fd.colCount).append(" columns, ").append(fd.stripes.size()).append(" stripes; ");
- for (StripeData stripe : fd.stripes) {
- if (stripe.data == null) continue;
- for (int i = 0; i < stripe.data.length; ++i) {
- LlapDataBuffer[][] colData = stripe.data[i];
- if (colData == null) continue;
- for (int j = 0; j < colData.length; ++j) {
- LlapDataBuffer[] streamData = colData[j];
- if (streamData == null) continue;
- for (int k = 0; k < streamData.length; ++k) {
- int newRc = streamData[k].incRef();
- if (newRc < 0) {
- ++fileEvicted;
- continue;
- }
- try {
- if (newRc > 1) { // We hold one refcount.
- ++fileLocked;
- } else {
- ++fileUnlocked;
- }
- } finally {
- streamData[k].decRef();
- }
- }
- }
- }
- }
- allLocked += fileLocked;
- allUnlocked += fileUnlocked;
- allEvicted += fileEvicted;
- sb.append("\n file " + e.getKey() + ": " + fileLocked + " locked, "
- + fileUnlocked + " unlocked, " + fileEvicted + " evicted");
- } finally {
- e.getValue().decRef();
- }
- }
- sb.append("\nSerDe cache summary: " + allLocked + " locked, "
- + allUnlocked + " unlocked, " + allEvicted + " evicted");
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleAllocator.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleAllocator.java
index 51eb34e..d8f59d1 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleAllocator.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleAllocator.java
@@ -32,10 +32,9 @@ public final class SimpleAllocator implements Allocator, BuddyAllocatorMXBean {
private final boolean isDirect;
private static Field cleanerField;
static {
+ ByteBuffer tmp = ByteBuffer.allocateDirect(1);
try {
- // TODO: To make it work for JDK9 use CleanerUtil from https://issues.apache.org/jira/browse/HADOOP-12760
- final Class<?> dbClazz = Class.forName("java.nio.DirectByteBuffer");
- cleanerField = dbClazz.getDeclaredField("cleaner");
+ cleanerField = tmp.getClass().getDeclaredField("cleaner");
cleanerField.setAccessible(true);
} catch (Throwable t) {
LlapIoImpl.LOG.warn("Cannot initialize DirectByteBuffer cleaner", t);
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
index af7cf3d..d1eee04 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/SimpleBufferManager.java
@@ -102,14 +102,4 @@ public class SimpleBufferManager implements BufferUsageManager, LowLevelCache {
public void notifyEvicted(MemoryBuffer buffer) {
throw new UnsupportedOperationException("Buffer manager doesn't have cache");
}
-
- @Override
- public String debugDumpForOom() {
- return "";
- }
-
- @Override
- public void debugDumpShort(StringBuilder sb) {
- // TODO Auto-generated method stub
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
index a662c75..22e5ee8 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java
@@ -76,8 +76,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.eclipse.jetty.rewrite.handler.Rule;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.server.ssl.SslSocketConnector;
import org.joda.time.DateTime;
import org.json.JSONException;
import org.json.JSONObject;
@@ -379,8 +378,7 @@ public class LlapServiceDriver {
LlapTezUtils.class, // llap-tez
LlapInputFormat.class, // llap-server
HiveInputFormat.class, // hive-exec
- SslContextFactory.class, // hive-common (https deps)
- Rule.class, // Jetty rewrite class
+ SslSocketConnector.class, // hive-common (https deps)
RegistryUtils.ServiceRecordMarshal.class, // ZK registry
// log4j2
com.lmax.disruptor.RingBuffer.class, // disruptor
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
index 2d0121c..8342067 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapSliderUtils.java
@@ -24,11 +24,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Clock;
-import org.apache.hadoop.yarn.util.SystemClock;
-import org.apache.slider.api.types.ApplicationDiagnostics;
import org.apache.slider.client.SliderClient;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionDestroyArgs;
@@ -62,60 +58,6 @@ public class LlapSliderUtils {
return sliderClient;
}
- public static ApplicationReport getAppReport(String appName, SliderClient sliderClient,
- long timeoutMs) throws
- LlapStatusServiceDriver.LlapStatusCliException {
- Clock clock = new SystemClock();
- long startTime = clock.getTime();
- long timeoutTime = timeoutMs < 0 ? Long.MAX_VALUE : (startTime + timeoutMs);
- ApplicationReport appReport = null;
-
- while (appReport == null) {
- try {
- appReport = sliderClient.getYarnAppListClient().findInstance(appName);
- if (timeoutMs == 0) {
- // break immediately if timeout is 0
- break;
- }
- // Otherwise sleep, and try again.
- if (appReport == null) {
- long remainingTime = Math.min(timeoutTime - clock.getTime(), 500l);
- if (remainingTime > 0) {
- Thread.sleep(remainingTime);
- } else {
- break;
- }
- }
- } catch (Exception e) { // No point separating IOException vs YarnException vs others
- throw new LlapStatusServiceDriver.LlapStatusCliException(
- LlapStatusServiceDriver.ExitCode.YARN_ERROR,
- "Failed to get Yarn AppReport", e);
- }
- }
- return appReport;
- }
-
- public static ApplicationDiagnostics getApplicationDiagnosticsFromYarnDiagnostics(
- ApplicationReport appReport, Logger LOG) {
- if (appReport == null) {
- return null;
- }
- String diagnostics = appReport.getDiagnostics();
- if (diagnostics == null || diagnostics.isEmpty()) {
- return null;
- }
- try {
- ApplicationDiagnostics appDiagnostics =
- ApplicationDiagnostics.fromJson(diagnostics);
- return appDiagnostics;
- } catch (IOException e) {
- LOG.warn(
- "Failed to parse application diagnostics from Yarn Diagnostics - {}",
- diagnostics);
- return null;
- }
- }
-
public static void startCluster(Configuration conf, String name,
String packageName, Path packageDir, String queue) {
LOG.info("Starting cluster with " + name + ", "
http://git-wip-us.apache.org/repos/asf/hive/blob/ed64a74e/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
index bd91495..b4aa430 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapStatusOptionsProcessor.java
@@ -38,7 +38,6 @@ public class LlapStatusOptionsProcessor {
private static final long DEFAULT_STATUS_REFRESH_INTERVAL_MS = 1 * 1000l; // 1 seconds wait until subsequent status
private static final long DEFAULT_WATCH_MODE_TIMEOUT_MS = 5 * 60 * 1000l; // 5 minutes timeout for watch mode
private static final float DEFAULT_RUNNING_NODES_THRESHOLD = 1.0f;
-
enum OptionConstants {
NAME("name", 'n', "LLAP cluster name", true),