You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2014/11/07 02:01:09 UTC
svn commit: r1637279 - in
/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc:
JDBCStatsAggregator.java JDBCStatsPublisher.java
JDBCStatsSetupConstants.java JDBCStatsUtils.java
Author: prasanthj
Date: Fri Nov 7 01:01:08 2014
New Revision: 1637279
URL: http://svn.apache.org/r1637279
Log:
HIVE-8735: statistics update can fail due to long paths (Sergey Shelukhin reviewed by Prasanth J)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java?rev=1637279&r1=1637278&r2=1637279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsAggregator.java Fri Nov 7 01:01:08 2014
@@ -134,6 +134,7 @@ public class JDBCStatsAggregator impleme
}
};
+ fileID = JDBCStatsUtils.truncateRowId(fileID);
String keyPrefix = Utilities.escapeSqlLike(fileID) + "%";
for (int failures = 0;; failures++) {
try {
@@ -147,7 +148,7 @@ public class JDBCStatsAggregator impleme
if (result.next()) {
retval = result.getLong(1);
} else {
- LOG.warn("Warning. Nothing published. Nothing aggregated.");
+ LOG.warn("Nothing published. Nothing aggregated.");
return null;
}
return Long.toString(retval);
@@ -217,6 +218,7 @@ public class JDBCStatsAggregator impleme
};
try {
+ rowID = JDBCStatsUtils.truncateRowId(rowID);
String keyPrefix = Utilities.escapeSqlLike(rowID) + "%";
PreparedStatement delStmt = Utilities.prepareWithRetry(conn,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java?rev=1637279&r1=1637278&r2=1637279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsPublisher.java Fri Nov 7 01:01:08 2014
@@ -139,7 +139,11 @@ public class JDBCStatsPublisher implemen
+ " stats: " + JDBCStatsUtils.getSupportedStatistics());
return false;
}
- LOG.info("Stats publishing for key " + fileID);
+ String rowId = JDBCStatsUtils.truncateRowId(fileID);
+ if (LOG.isInfoEnabled()) {
+ String truncateSuffix = (rowId != fileID) ? " (from " + fileID + ")" : ""; // object equality
+ LOG.info("Stats publishing for key " + rowId + truncateSuffix);
+ }
Utilities.SQLCommand<Void> execUpdate = new Utilities.SQLCommand<Void>() {
@Override
@@ -153,7 +157,7 @@ public class JDBCStatsPublisher implemen
for (int failures = 0;; failures++) {
try {
- insStmt.setString(1, fileID);
+ insStmt.setString(1, rowId);
for (int i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) {
insStmt.setString(i + 2, stats.get(supportedStatistics.get(i)));
}
@@ -172,10 +176,10 @@ public class JDBCStatsPublisher implemen
for (i = 0; i < JDBCStatsUtils.getSupportedStatistics().size(); i++) {
updStmt.setString(i + 1, stats.get(supportedStatistics.get(i)));
}
- updStmt.setString(supportedStatistics.size() + 1, fileID);
+ updStmt.setString(supportedStatistics.size() + 1, rowId);
updStmt.setString(supportedStatistics.size() + 2,
stats.get(JDBCStatsUtils.getBasicStat()));
- updStmt.setString(supportedStatistics.size() + 3, fileID);
+ updStmt.setString(supportedStatistics.size() + 3, rowId);
Utilities.executeWithRetry(execUpdate, updStmt, waitWindow, maxRetries);
return true;
} catch (SQLRecoverableException ue) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java?rev=1637279&r1=1637278&r2=1637279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsSetupConstants.java Fri Nov 7 01:01:08 2014
@@ -34,4 +34,7 @@ public final class JDBCStatsSetupConstan
public static final String PART_STAT_RAW_DATA_SIZE_COLUMN_NAME = "RAW_DATA_SIZE";
+ // 255 is an old value that we will keep for now; it can be increased to 4000; limits are
+ // MySQL - 65535, SQL Server - 8000, Oracle - 4000, Derby - 32762, Postgres - large.
+ public static final int ID_COLUMN_VARCHAR_SIZE = 255;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java?rev=1637279&r1=1637278&r2=1637279&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/stats/jdbc/JDBCStatsUtils.java Fri Nov 7 01:01:08 2014
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.util.hash.MurmurHash;
public class JDBCStatsUtils {
@@ -124,9 +125,10 @@ public class JDBCStatsUtils {
* Prepares CREATE TABLE query
*/
public static String getCreate(String comment) {
- String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName() +
- " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, " +
- JDBCStatsUtils.getIdColumnName() + " VARCHAR(255) PRIMARY KEY ";
+ String create = "CREATE TABLE /* " + comment + " */ " + JDBCStatsUtils.getStatTableName()
+ + " (" + getTimestampColumnName() + " TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "
+ + JDBCStatsUtils.getIdColumnName() + " VARCHAR("
+ + JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE + ") PRIMARY KEY ";
for (int i = 0; i < supportedStats.size(); i++) {
create += ", " + getStatColumnName(supportedStats.get(i)) + " BIGINT ";
}
@@ -191,4 +193,13 @@ public class JDBCStatsUtils {
return delete;
}
+ /**
+ * Make sure the row ID fits into the row ID column in the table.
+ * @param rowId Row ID.
+ * @return Resulting row ID truncated to correct length, if necessary.
+ */
+ public static String truncateRowId(String rowId) {
+ return (rowId.length() <= JDBCStatsSetupConstants.ID_COLUMN_VARCHAR_SIZE)
+ ? rowId : Integer.toHexString(MurmurHash.getInstance().hash(rowId.getBytes()));
+ }
}