You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@hive.apache.org by "veghlaci05 (via GitHub)" <gi...@apache.org> on 2023/06/02 12:45:35 UTC

[GitHub] [hive] veghlaci05 opened a new pull request, #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

veghlaci05 opened a new pull request, #4384:
URL: https://github.com/apache/hive/pull/4384

   ### What changes were proposed in this pull request?
   Standardization and simplification of JDBC related code in CompactionTxnHandler and TxnHandler
   
   Notes for reviewers:
   This PR is currently a draft only, and supposet to show how the code could be redesigned using spring-jdbc. The aim is to get early feedback from other community members if the proposed approach is OK to go forward with.
   
   ### Why are the changes needed?
   These classes are getting too complex, and a source of tech-debt accumulation
   
   
   ### Does this PR introduce _any_ user-facing change?
   No
   
   ### How was this patch tested?
   Through unit and integratoin tests


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310035924


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,38 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).execute(jdbcTemplate);

Review Comment:
   1st: While it was a great idea for QueryHandlers and ParameterizedCommands, for the TransactionalFunction it would look like: 
   ```
     public <R> R execute(TransactionalFunction<R> function) throws SQLException, MetaException {
       function.execute(this);
     }
   
   ```
   In this case it doesn't have much added value.
   
   2st: fixed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337258146


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6396,52 +6430,45 @@ private AbortCompactionResponseElement getAbortCompactionResponseElement(long co
   }
 
   @RetrySemantics.SafeToRetry
-  public AbortCompactionResponseElement abortCompaction(CompactionInfo compactionInfo) throws MetaException {

Review Comment:
   compact() is also in TxnHandler, and this method aborts the compaction created by the other one. However I'm also not strictly tied to the idea that it must be in TxnHandler.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309036735


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -438,6 +467,9 @@ public void setConf(Configuration conf){
       LOG.error(msg);
       throw new RuntimeException(e);
     }
+
+    retryHandler = new RetryHandler();
+    retryHandler.init(conf, jdbcTemplate);    

Review Comment:
   could this be part of constructor?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309107696


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1603,21 +1645,24 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
         }
 
         if (!isHiveReplTxn) {
-          createCommitNotificationEvent(dbConn, txnid , txnType);
+          createCommitNotificationEvent(jdbcTemplate, txnid , txnType);

Review Comment:
   should it be dataSourceWrapper.getConnection()?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309116024


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -3645,6 +3678,38 @@ long generateCompactionQueueId(Statement stmt) throws SQLException, MetaExceptio
     }
   }
 
+  long generateCompactionQueueId() throws MetaException {
+    // Get the id for the next entry in the queue
+    String sql = sqlGenerator.addForUpdateClause("SELECT \"NCQ_NEXT\" FROM \"NEXT_COMPACTION_QUEUE_ID\"");

Review Comment:
   would that be executed in diff transaction context than the caller method?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1737538638

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1624412780

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [83 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1260687967


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionCandidate.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+
+public class CompactionCandidate extends CompactionInfoBase {
+
+  public boolean tooManyAborts;

Review Comment:
   You mean flagged enum? `tooManyAborts` and `hasOldAborts` are independent variables. You can have too many aborts without having too old ones, etc.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293062497


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +93,12 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
+    Set<CompactionInfo> candidates = new HashSet<>(new CompactionCandidateHandler(lastChecked, 

Review Comment:
   could we refactor to something like
   ````
   CompactionCandidateHandler.getInstance()
       .execute(request, dataSourceWrapper)
   ````



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293117112


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -58,114 +68,18 @@
  * methods are not available through the thrift interface.
  */
 class CompactionTxnHandler extends TxnHandler {
-  private static final String CLASS_NAME = CompactionTxnHandler.class.getName();
-  private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
-  private static final String DB_FAILED_TO_CONNECT = "Unable to connect to transaction database: ";
-
-  private static DataSource connPoolCompaction;
-
-  private static final String SELECT_COMPACTION_QUEUE_BY_TXN_ID =
-      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-          + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-          + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-          + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-          + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-          + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" "
-          + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?";
-  private static final String SELECT_COMPACTION_METRICS_CACHE_QUERY =
-      "SELECT \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM \"COMPACTION_METRICS_CACHE\" " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ?";
-  private static final String NO_SELECT_COMPACTION_METRICS_CACHE_FOR_TYPE_QUERY =
-      "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM " +
-      "\"COMPACTION_METRICS_CACHE\" WHERE \"CMC_METRIC_TYPE\" = ? ORDER BY \"CMC_METRIC_VALUE\" DESC";
-  private static final String UPDATE_COMPACTION_METRICS_CACHE_QUERY =
-      "UPDATE \"COMPACTION_METRICS_CACHE\" SET \"CMC_METRIC_VALUE\" = ?, \"CMC_VERSION\" = ? " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ? " +
-          "AND \"CMC_VERSION\" = ?";
-  private static final String INSERT_COMPACTION_METRICS_CACHE_QUERY =
-      "INSERT INTO \"COMPACTION_METRICS_CACHE\" ( " +
-          "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_TYPE\", \"CMC_METRIC_VALUE\", " +
-          "\"CMC_VERSION\" ) VALUES (?, ?, ?, ?, ?, ?)";
-  private static final String DELETE_COMPACTION_METRICS_CACHE_QUERY =
-      "DELETE FROM \"COMPACTION_METRICS_CACHE\" WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? " +
-          "AND \"CMC_METRIC_TYPE\" = ?";
-
-  private static final String DELETE_FAILED_TXNS_SQL =
-      "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
-          "AND (\"TXN_STATE\" = " + TxnStatus.ABORTED + " OR \"TXN_STATE\" = " + TxnStatus.COMMITTED + ") " +
-          "AND \"TXN_ID\" < ?";
-
-  // Three inner sub-queries which are under left-join to fetch the required data for aborted txns.
-  private static final String SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY =
-          " \"res1\".\"TC_DATABASE\" AS \"DB\", \"res1\".\"TC_TABLE\" AS \"TBL\", \"res1\".\"TC_PARTITION\" AS \"PART\", " +
-          " \"res1\".\"MIN_TXN_START_TIME\" AS \"MIN_TXN_START_TIME\", \"res1\".\"ABORTED_TXN_COUNT\" AS \"ABORTED_TXN_COUNT\", " +
-          " \"res2\".\"MIN_OPEN_WRITE_TXNID\" AS \"MIN_OPEN_WRITE_TXNID\", \"res3\".\"RETRY_RETENTION\" AS \"RETRY_RETENTION\", " +
-          " \"res3\".\"ID\" AS \"RETRY_CQ_ID\" " +
-      " FROM " +
-          // First sub-query - Gets the aborted txns with min txn start time, number of aborted txns
-          // for corresponding db, table, partition.
-          " ( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TXN_STARTED\") AS \"MIN_TXN_START_TIME\", " +
-          " COUNT(*) AS \"ABORTED_TXN_COUNT\" FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" %s ) \"res1\" " +
-      " LEFT JOIN" +
-          // Second sub-query - Gets the min open txn id for corresponding db, table, partition.
-          "( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TC_TXNID\") AS \"MIN_OPEN_WRITE_TXNID\" " +
-          " FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" ) \"res2\"" +
-      " ON \"res1\".\"TC_DATABASE\" = \"res2\".\"TC_DATABASE\"" +
-      " AND \"res1\".\"TC_TABLE\" = \"res2\".\"TC_TABLE\"" +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res2\".\"TC_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res2\".\"TC_PARTITION\" IS NULL)) " +
-      " LEFT JOIN " +
-          // Third sub-query - Gets the retry entries for corresponding db, table, partition.
-          "( SELECT \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", MAX(\"CQ_ID\") AS \"ID\", " +
-          " MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RETENTION\", " +
-          " MIN(\"CQ_COMMIT_TIME\") - %s + MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RECORD_CHECK\" FROM \"COMPACTION_QUEUE\" " +
-          " WHERE \"CQ_TYPE\" = " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"res3\" " +
-      " ON \"res1\".\"TC_DATABASE\" = \"res3\".\"CQ_DATABASE\" " +
-      " AND \"res1\".\"TC_TABLE\" = \"res3\".\"CQ_TABLE\" " +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res3\".\"CQ_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res3\".\"CQ_PARTITION\" IS NULL))" +
-      " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL";
-
-  private static final String DELETE_CQ_ENTRIES = "DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-
-  private static final String DELETE_CQ_AND_ABORT_ENTRIES = DELETE_CQ_ENTRIES +
-      " OR ( \"CQ_DATABASE\" = ? AND \"CQ_TABLE\" = ? " +
-      " AND \"CQ_PARTITION\" %s AND \"CQ_TYPE\" = " +
-      quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) + " )";
-
-  private static final String INSERT_ABORT_RETRY_ENTRY_INTO_COMPACTION_QUEUE =
-      "INSERT INTO \"COMPACTION_QUEUE\" (\"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
-      " \"CQ_TYPE\", \"CQ_STATE\", \"CQ_RETRY_RETENTION\", \"CQ_ERROR_MESSAGE\", \"CQ_COMMIT_TIME\") " +
-      " VALUES (?, ?, ?, ?, ?, ?, ?, ?, %s)";
+  
+  private static final Logger LOG = LoggerFactory.getLogger(CompactionTxnHandler.class.getName());
 
   public CompactionTxnHandler() {
   }
 
-  @Override
-  public void setConf(Configuration conf) {
-    super.setConf(conf);
-    synchronized (CompactionTxnHandler.class) {
-      if (connPoolCompaction == null) {
-        int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);

Review Comment:
   It's moved to TxnHandler: from now on all pools are handled by DataSourceWrapper.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293081977


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +141,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    ParameterizedCommand.execute(dataSourceWrapper, 

Review Comment:
   Originally this stlye was used everywhere, I have no strong feelings about it, I can restore it in QueryHandler also.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] aturoczy commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "aturoczy (via GitHub)" <gi...@apache.org>.
aturoczy commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1256337086


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionCandidate.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+
+public class CompactionCandidate extends CompactionInfoBase {
+
+  public boolean tooManyAborts;

Review Comment:
   Would not be better to have an Enum for this type of issues?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * Supports building resilient APIs by automatically retrying failed function calls.
+ */
+public class RetryHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);

Review Comment:
   Is this still need for the latest Derby? I have a bad feeling that there is a specific logic around a provider



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1623742779

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [83 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1229822489


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state

Review Comment:
   The state is a not null field



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] InvisibleProgrammer commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "InvisibleProgrammer (via GitHub)" <gi...@apache.org>.
InvisibleProgrammer commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1230707481


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state
+    try {

Review Comment:
   The column based mapper works only if the column indexes are matching with the column order, which is quite fragile. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] aturoczy commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "aturoczy (via GitHub)" <gi...@apache.org>.
aturoczy commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1227696507


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state
+    try {

Review Comment:
   There is no object relation mapping in this case? For my C# eyes this getString(index) hurts :) 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1260685117


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java:
##########
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * Supports building resilient APIs by automatically retrying failed function calls.
+ */
+public class RetryHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);

Review Comment:
   I think it is still required, I faced deadlock issues with Derby elsewhere in the code recently and had to use locking to solve it. However, I'm not completely sure to be honest.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293060235


##########
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java:
##########
@@ -742,12 +740,18 @@ public void testRevokeTimedOutWorkers() throws Exception {
   public void testFindPotentialCompactions() throws Exception {
     // Test that committing unlocks
     long txnid = openTxn();
-
-    List<LockComponent> components = new ArrayList<>();
-
-    components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "mytable", null, DataOperationType.UPDATE));
-    components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "yourtable", "mypartition=myvalue", DataOperationType.UPDATE));
-
+    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB,

Review Comment:
   This is a merge issue, I'll fix it.



##########
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java:
##########
@@ -999,62 +1003,6 @@ public void testEnqueueTimeThroughLifeCycle() throws Exception {
     checkEnqueueTime(enqueueTime);
   }
 
-  @Test
-  public void testFindPotentialCompactions_limitFetchSize() throws Exception {

Review Comment:
   This is a merge issue, I'll fix it.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293071008


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +141,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    ParameterizedCommand.execute(dataSourceWrapper, 

Review Comment:
   i like this style, would be even better if we declare jdbcTemplate embedding dataSourceWrapper and exposing execute command
   ````
   jdbcTemplate.execute(
       "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = :state, \"CQ_WORKER_ID\" = NULL WHERE 
       \"CQ_ID\" = :id",
           new MapSqlParameterSource()
               .addValue("state", Character.toString(READY_FOR_CLEANING), Types.CHAR)
               .addValue("id", info.id),
           ParameterizedCommand.EXACTLY_ONE_ROW))
   ````



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293109026


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,38 +261,22 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction)) {
-        try (PreparedStatement stmt = dbConn.prepareStatement(DELETE_FAILED_TXNS_SQL)) {
-          //Aborted and committed are terminal states, so nothing about the txn can change
-          //after that, so READ COMMITTED is sufficient.
-          /*
-           * Only delete aborted / committed transaction in a way that guarantees two things:
-           * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
-           * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
-           */
-          long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId(dbConn);
-          stmt.setLong(1, lowWaterMark);
-          LOG.debug("Going to execute query <{}>", DELETE_FAILED_TXNS_SQL);
-          int rc = stmt.executeUpdate();
-          LOG.debug("Removed {} empty Aborted and Committed transactions from TXNS", rc);
-          LOG.debug("Going to commit");
-          dbConn.commit();
-        } catch (SQLException e) {
-          LOG.error("Unable to delete from txns table " + e.getMessage());
-          LOG.debug("Going to rollback");
-          rollbackDBConn(dbConn);
-          checkRetryable(e, "cleanEmptyAbortedTxns");
-          throw new MetaException("Unable to delete from txns table " + e.getMessage());
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e, "cleanEmptyAbortedTxns");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedAndCommittedTxns();
-    }
+    //Aborted and committed are terminal states, so nothing about the txn can change
+    //after that, so READ COMMITTED is sufficient.
+    /*
+     * Only delete aborted / committed transaction in a way that guarantees two things:
+     * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
+     * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
+     */
+    long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+    String query = "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
+        "AND (\"TXN_STATE\" = :abortedState OR \"TXN_STATE\" = :committedState) AND \"TXN_ID\" < :txnId";
+
+    LOG.debug("Going to execute query <{}>", query);
+    dataSourceWrapper.getJdbcTemplate().update(query, new MapSqlParameterSource()

Review Comment:
   Sure, we can. for the very simple cases I did not create a separate class, but to be consistent, I can create for these too.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,39 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).call(dataSourceWrapper);
   }
-
+  
   /**
    * Clean up entries from TXN_TO_WRITE_ID table less than min_uncommited_txnid as found by
    * min(max(TXNS.txn_id), min(WRITE_SET.WS_COMMIT_ID), min(Aborted TXNS.txn_id)).
    */
   @Override
   @RetrySemantics.SafeToRetry
   public void cleanTxnToWriteIdTable() throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-
-      try {
-        long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
-
-        // We query for minimum values in all the queries and they can only increase by any concurrent
-        // operations. So, READ COMMITTED is sufficient.
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
-        // If there are no txns which are currently open or aborted in the system, then current value of
-        // max(TXNS.txn_id) could be min_uncommitted_txnid.
-        String s = "SELECT MIN(\"RES\".\"ID\") AS \"ID\" FROM (" +
-          " SELECT MAX(\"TXN_ID\") + 1 AS \"ID\" FROM \"TXNS\"" +
-          (useMinHistoryLevel ? "" :
-          "   UNION" +
-          " SELECT MIN(\"WS_TXNID\") AS \"ID\" FROM \"WRITE_SET\"") +
-          "   UNION" +
-          " SELECT MIN(\"TXN_ID\") AS \"ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          (useMinHistoryLevel ? "" :
-          "   OR \"TXN_STATE\" = " + TxnStatus.OPEN) +
-          " ) \"RES\"";
+    long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
 
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
-        }
-        long minUncommitedTxnid = Math.min(rs.getLong(1), minTxnIdSeenOpen);
-
-        // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
-        // to cleanup the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
-        s = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < " + minUncommitedTxnid;
-        LOG.debug("Going to execute delete <{}>", s);
-        int rc = stmt.executeUpdate(s);
-        LOG.info("Removed {} rows from TXN_TO_WRITE_ID with Txn Low-Water-Mark: {}", rc, minUncommitedTxnid);
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from TXN_TO_WRITE_ID table " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "cleanTxnToWriteIdTable");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      cleanTxnToWriteIdTable();
+    // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
+    // If there are no txns which are currently open or aborted in the system, then current value of
+    // max(TXNS.txn_id) could be min_uncommitted_txnid.
+    Long minTxnId = new MinUncommittedTxnIdHandler(useMinHistoryLevel).execute(dataSourceWrapper);
+    if (minTxnId == null) {
+      throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
     }
+    long minUncommitedTxnid = Math.min(minTxnId, minTxnIdSeenOpen);
+
+    // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
+    // to clean up the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
+    String sql = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < :txnId";
+    LOG.debug("Going to execute delete <{}>", sql);
+    int rc = dataSourceWrapper.getJdbcTemplate().update(sql, new MapSqlParameterSource("txnId", minUncommitedTxnid));

Review Comment:
   Sure, we can. for the very simple cases I did not create a separate class, but to be consistent, I can create for these too.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1336981979


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +112,10 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
-    }
+    return new FindPotentialCompactionsFunction(
+        MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
+        abortedThreshold, abortedTimeThreshold, lastChecked,
+        !MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)).execute(jdbcResource);

Review Comment:
   please move `execute` on new line



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337172597


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +158,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    jdbcResource.execute(
+        "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = :state, \"CQ_WORKER_ID\" = NULL WHERE \"CQ_ID\" = :id",

Review Comment:
   This is an update, here IS NULL cannot be used.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337248003


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6396,52 +6430,45 @@ private AbortCompactionResponseElement getAbortCompactionResponseElement(long co
   }
 
   @RetrySemantics.SafeToRetry
-  public AbortCompactionResponseElement abortCompaction(CompactionInfo compactionInfo) throws MetaException {

Review Comment:
   any idea why this method is not in CompactioinTxnHandler



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309923004


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -615,57 +190,19 @@ public void markCleanerStart(CompactionInfo info) throws MetaException {
   @RetrySemantics.ReadOnly
   public void clearCleanerStart(CompactionInfo info) throws MetaException {
     LOG.debug("Running clearCleanerStart with CompactionInfo: {}", info);
-
-    try {
-      Connection dbConn = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        setCleanerStart(dbConn, info, -1L);
-      } catch (SQLException e) {
-        LOG.error("Unable to clear the cleaner start time for compaction record  " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "clearCleanerStart(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      clearCleanerStart(info);
-    }
+    setCleanerStart(info, -1L);
   }
 
-  private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timestamp)
-      throws RetryException, SQLException {
-    long id = info.id;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      String query = "" +
-          " UPDATE " +
-          "  \"COMPACTION_QUEUE\" " +
-          " SET " +
-          "  \"CQ_CLEANER_START\" = " + timestamp +
-          " WHERE " +
-          "  \"CQ_ID\" = " + id +
-          " AND " +
-          "  \"CQ_STATE\"='" + READY_FOR_CLEANING + "'";
-
-      pStmt = dbConn.prepareStatement(query);
-      LOG.debug("Going to execute update <{}> for CQ_ID={}", query, id);
-      int updCount = pStmt.executeUpdate();
-      if (updCount != 1) {
-        LOG.error("Unable to update compaction record: {}.  Update count={}", info, updCount);
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-      } else {
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      }
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+  private void setCleanerStart(CompactionInfo info, Long timestamp) throws MetaException {
+    jdbcTemplate.execute(
+        " UPDATE \"COMPACTION_QUEUE\" " +

Review Comment:
   This will be optimized by the compiler as there are no dynamic parts in the concatenation. I would prefer keeping the query itself inside or close to the jdbcTemplate call as it improves readability (no need to navigate back and forth between the constant and the method/class) . However I'm not strictly tied to it. If you prefer I can also create constants.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309093634


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1441,6 +1480,7 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
             // corresponding open txn event.
             LOG.info("Target txn id is missing for source txn id : {} and repl policy {}", sourceTxnId,
                 rqst.getReplPolicy());
+            jdbcTemplate.rollback();

Review Comment:
   do you need to call rollback explicitly even when nothing changed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309786856


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;
+  }
+
+  /**
+   * @return Returns true if the current {@link Thread} has an associated {@link RetryContext}, false otherwise.
+   */
+  boolean hasRetryContext() {
+    return threadLocal.get() != null;
+  }
+
+  /**
+   * @return Returns the {@link NamedParameterJdbcTemplate} associated with the current {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public NamedParameterJdbcTemplate getJdbcTemplate() {
+    return jdbcTemplates.get(RetryContext.getRetryContext().datasource);
+  }
+
+  /**
+   * Gets a connection to the {@link DataSource} associated with the current {@link RetryContext}. Ensures that the same 
+   * instance is returned all the time within a particular transaction. Can be called only within an established 
+   * {@link RetryContext}. 
+   * @return Returns a connection to the datasource.
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public Connection getConnection() {
+    return DataSourceUtils.getConnection(dataSources.get(RetryContext.getRetryContext().datasource));
+  }
+
+  /**
+   * @return Returns a {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DatabaseProduct getDatabaseProduct() {
+    return databaseProduct;
+  }
+
+  /**
+   * <p>Establishes a {@link RetryContext} on the current {@link Thread}: Creates a new transaction with the given 
+   * {@link TransactionDefinition} using an internal {@link PlatformTransactionManager} associated with the reffered 
+   * {@link DataSource}. Returns a {@link TransactionStatus} representing the created transaction. The returned 
+   * {@link TransactionStatus} instance can be used for manual transaction handling.</p> 
+   * <br>
+   * <p>!!!! Please note that if there already is a {@link RetryContext} established, this call will create a nested</p>
+   * {@link RetryContext} (and transaction).</b>  
+   * @param definition The {@link TransactionDefinition} to use for creating a new transaction.
+   * @param dataSource The identifier of the {@link DataSource} to use
+   * @return Returns a {@link TransactionStatus} object representing the created transaction.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#getTransaction(TransactionDefinition)}
+   */
+  public TransactionStatus getTransactionWithinRetryContext(TransactionDefinition definition, String dataSource) throws TransactionException {
+    TransactionStatus status = transactionManagers.get(dataSource).getTransaction(definition);
+    RetryContext.setRetryContext(status, dataSource);
+    return status;
+  }
+
+  /**
+   * Commits the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#commit(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void commit() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();
+      transactionManagers.get(context.datasource).commit(context.transactionStatus);
+    } finally {
+      RetryContext.clearRetryContext();
+    }
+  }
+
+  /**
+   * Rollbacks the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#rollback(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void rollback() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();

Review Comment:
   context could be nested, how do we calculate the transactionStatus?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310052543


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5755,80 +5807,86 @@ private void timeOutLocks(Connection dbConn) {
    */
   @RetrySemantics.Idempotent
   public void performTimeOuts() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
     try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-      //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
-      //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
-      //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
-      //would be to make sure someone cannot heartbeat one of these txns at the same time.
-      //The attempt to heartbeat would block and fail immediately after it's unblocked.
-      //With current (RC + multiple txns) implementation it is possible for someone to send
-      //heartbeat at the very end of the expire interval, and just after the Select from TXNS
-      //is made, in which case heartbeat will succeed but txn will still be Aborted.
-      //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
-      //timely way.
-      timeOutLocks(dbConn);
-      while(true) {
-        stmt = dbConn.createStatement();
-        String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " AND (" +
-                "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
-             " OR " +
-                " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
-             ")";
-        //safety valve for extreme cases
-        s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if(!rs.next()) {
-          return;//no more timedout txns
-        }
-        List<List<Long>> timedOutTxns = new ArrayList<>();
-        List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-        timedOutTxns.add(currentBatch);
-        do {
-          if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
-            currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-            timedOutTxns.add(currentBatch);
-          }
-          currentBatch.add(rs.getLong(1));
-        } while(rs.next());
-        dbConn.commit();
-        close(rs, stmt, null);
-        int numTxnsAborted = 0;
-        for(List<Long> batchToAbort : timedOutTxns) {
-          if (abortTxns(dbConn, batchToAbort, true, false, false, TxnErrorMsg.ABORT_TIMEOUT) == batchToAbort.size()) {
-            dbConn.commit();
-            numTxnsAborted += batchToAbort.size();
-            //todo: add TXNS.COMMENT filed and set it to 'aborted by system due to timeout'
-          }
-          else {
-            //could not abort all txns in this batch - this may happen because in parallel with this
-            //operation there was activity on one of the txns in this batch (commit/abort/heartbeat)
-            //This is not likely but may happen if client experiences long pause between heartbeats or
-            //unusually long/extreme pauses between heartbeat() calls and other logic in checkLock(),
-            //lock(), etc.
-            dbConn.rollback();
+      retryHandler.executeWithoutRetry(
+          new RetryCallProperties()
+              .withCallerId("performTimeOuts()")
+              .withDataSource(POOL_TX)
+              .withExceptionSupplier(e -> new MetaException("Aborting timed out transactions failed due to " + RetryHandler.getMessage(e))),
+          (DataSourceWrapper dataSourceWrapper) -> {
+            //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
+            //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
+            //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
+            //would be to make sure someone cannot heartbeat one of these txns at the same time.
+            //The attempt to heartbeat would block and fail immediately after it's unblocked.
+            //With current (RC + multiple txns) implementation it is possible for someone to send
+            //heartbeat at the very end of the expiry interval, and just after the Select from TXNS
+            //is made, in which case heartbeat will succeed but txn will still be Aborted.
+            //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
+            //timely way.
+            timeOutLocks();
+            while (true) {
+              String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
+                  " AND (" +
+                  "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
+                  " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
+                  " OR " +
+                  " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
+                  " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
+                  ")";
+              //safety valve for extreme cases
+              s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
+
+              LOG.debug("Going to execute query <{}>", s);
+              List<List<Long>> timedOutTxns = dataSourceWrapper.getJdbcTemplate().query(s, rs -> {
+                List<List<Long>> txnbatch = new ArrayList<>();
+                List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+                while (rs.next()) {
+                  currentBatch.add(rs.getLong(1));
+                  if (currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
+                    txnbatch.add(currentBatch);
+                    currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+                  }
+                }
+                if (currentBatch.size() > 0) {
+                  txnbatch.add(currentBatch);
+                }
+                return txnbatch;
+              });
+              //noinspection DataFlowIssue
+              if (timedOutTxns.size() == 0) {
+                return Void.TYPE;
+              }
+
+              TransactionStatus status = dataSourceWrapper.getTransactionStatus();
+              Object savePoint = status.createSavepoint();
+
+              int numTxnsAborted = 0;
+              for (List<Long> batchToAbort : timedOutTxns) {
+                status.releaseSavepoint(savePoint);
+                savePoint = status.createSavepoint();

Review Comment:
   is this really required, how much overhead would it create if db is overloaded



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310059255


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2575,11 +2601,17 @@ protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException,
           minOpenTxn = Long.MAX_VALUE;
         }
       }
+    } catch (SQLException e) {
+      throw new UncategorizedSQLException(null, null, e);
+    }
+    jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);

Review Comment:
   do we open a new connection here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309111795


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2502,40 +2546,22 @@ public void seedTxnId(SeedTxnIdRequest rqst) throws MetaException {
 
   @Override
   @RetrySemantics.Idempotent
-  public void addWriteNotificationLog(ListenerEvent acidWriteEvent)
-          throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        //Idempotent case is handled by notify Event
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners,
-                acidWriteEvent instanceof AcidWriteEvent ? EventMessage.EventType.ACID_WRITE
-                        : EventMessage.EventType.BATCH_ACID_WRITE,
-                acidWriteEvent, dbConn, sqlGenerator);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback: ", e);
-        rollbackDBConn(dbConn);
-        if (isDuplicateKeyError(e)) {
-          // in case of key duplicate error, retry as it might be because of race condition
-          if (waitForRetry("addWriteNotificationLog(" + acidWriteEvent + ")", e.getMessage())) {
-            throw new RetryException();
-          }
-          retryNum = 0;
-          throw new MetaException(e.getMessage());
+  public void addWriteNotificationLog(ListenerEvent acidWriteEvent) throws MetaException {
+    retryHandler.executeWithRetry(new RetryCallProperties()

Review Comment:
   why don't we use jdbcTemplate here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310065980


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +93,12 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
+    Set<CompactionInfo> candidates = new HashSet<>(jdbcTemplate.execute(
+        new CompactionCandidateHandler(lastChecked, MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE))));
+    if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310082117


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,38 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).execute(jdbcTemplate);
   }
-
+  
   /**
    * Clean up entries from TXN_TO_WRITE_ID table less than min_uncommited_txnid as found by
    * min(max(TXNS.txn_id), min(WRITE_SET.WS_COMMIT_ID), min(Aborted TXNS.txn_id)).
    */
   @Override
   @RetrySemantics.SafeToRetry
   public void cleanTxnToWriteIdTable() throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-
-      try {
-        long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
-
-        // We query for minimum values in all the queries and they can only increase by any concurrent
-        // operations. So, READ COMMITTED is sufficient.
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
-        // If there are no txns which are currently open or aborted in the system, then current value of
-        // max(TXNS.txn_id) could be min_uncommitted_txnid.
-        String s = "SELECT MIN(\"RES\".\"ID\") AS \"ID\" FROM (" +
-          " SELECT MAX(\"TXN_ID\") + 1 AS \"ID\" FROM \"TXNS\"" +
-          (useMinHistoryLevel ? "" :
-          "   UNION" +
-          " SELECT MIN(\"WS_TXNID\") AS \"ID\" FROM \"WRITE_SET\"") +
-          "   UNION" +
-          " SELECT MIN(\"TXN_ID\") AS \"ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          (useMinHistoryLevel ? "" :
-          "   OR \"TXN_STATE\" = " + TxnStatus.OPEN) +
-          " ) \"RES\"";
+    long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
 
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
-        }
-        long minUncommitedTxnid = Math.min(rs.getLong(1), minTxnIdSeenOpen);
-
-        // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
-        // to cleanup the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
-        s = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < " + minUncommitedTxnid;
-        LOG.debug("Going to execute delete <{}>", s);
-        int rc = stmt.executeUpdate(s);
-        LOG.info("Removed {} rows from TXN_TO_WRITE_ID with Txn Low-Water-Mark: {}", rc, minUncommitedTxnid);
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from TXN_TO_WRITE_ID table " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "cleanTxnToWriteIdTable");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      cleanTxnToWriteIdTable();
+    // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
+    // If there are no txns which are currently open or aborted in the system, then current value of
+    // max(TXNS.txn_id) could be min_uncommitted_txnid.
+    Long minTxnId = jdbcTemplate.execute(new MinUncommittedTxnIdHandler(useMinHistoryLevel));
+    if (minTxnId == null) {

Review Comment:
   Fixed, created a new class: CleanTxnToWriteIdTableFunction



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1296882616


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java:
##########
@@ -131,6 +134,21 @@ enum MUTEX_KEY {
 
   int TIMED_OUT_TXN_ABORT_BATCH_SIZE = 50000;
 
+  String POOL_TX = "txnhandler";
+  String POOL_MUTEX = "mutex";
+  String POOL_COMPACTOR = "compactor";

Review Comment:
   TxnStore is an interface so these are implicitly `public static final`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] kasakrisz commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "kasakrisz (via GitHub)" <gi...@apache.org>.
kasakrisz commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1221543052


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/AbortedTxnCleaner.java:
##########
@@ -99,7 +100,7 @@ private void clean(CompactionInfo info, long minOpenWriteTxn, boolean metricsEna
       if (isNull(t)) {
         // The table was dropped before we got around to cleaning it.
         LOG.info("Unable to find table {}, assuming it was dropped.", info.getFullTableName());
-        txnHandler.markCleaned(info, true);
+        txnHandler.markCleaned((CompactionInfo) info, true);

Review Comment:
   Is it possible to change `markCleaned` to take `AbortTxnInfo` instead of `CompactionInfo`?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/RetryHandler.java:
##########
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+public class RetryHandler {
+
+  @FunctionalInterface
+  public interface VoidRetryFunction {
+
+    void call(TransactionStatus status) throws SQLException, MetaException;
+
+  }
+
+  @FunctionalInterface
+  public interface RetryFunction<Result> {
+
+    Result call(TransactionStatus status) throws SQLException, MetaException;
+
+  }
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+
+
+  private DatabaseProduct dbProduct;
+
+  private long deadlockRetryInterval;
+  private long retryInterval;
+
+  private int deadlockCnt;
+  private int retryLimit;
+  private int retryNum;
+  
+  private Configuration conf;
+
+
+  public static String getMessage(Exception ex) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(ex.getMessage());
+    if (ex instanceof SQLException) {
+      SQLException sqlEx = (SQLException)ex; 
+      sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")");
+    }
+    return sb.toString();
+  }  
+  
+  public void init(Configuration conf, DatabaseProduct dbProduct){
+    this.conf = conf;
+    this.dbProduct = dbProduct;
+
+    retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+  }
+  
+  public <Result> Result executeWithRetry(CallProperties properties, RetryFunction<Result> function) throws MetaException {
+    return executeWithRetryInternal(properties, function, true);
+  }
+
+  public void executeWithRetry(CallProperties properties, VoidRetryFunction function) throws MetaException {
+    executeWithRetryInternal(properties, (TransactionStatus status) -> { 
+      function.call(status); 
+      return null;
+    }, true);
+  }
+
+  public  <Result> Result executeWithoutRetry(CallProperties properties, RetryFunction<Result> function) throws MetaException {
+    return executeWithRetryInternal(properties, (TransactionStatus status) -> {
+      function.call(status);
+      return null;

Review Comment:
   Should the return value of the function be returned instead if `null`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1665984217

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [88 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1285506163


##########
service/pom.xml:
##########
@@ -221,6 +221,10 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-core</artifactId>

Review Comment:
   why is this change?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1672027066

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [76 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288086398


##########
service/pom.xml:
##########
@@ -221,6 +221,10 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-core</artifactId>

Review Comment:
   spring-core is already added to metastore-server, why it's needed here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1679303902

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [66 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293071579


##########
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java:
##########
@@ -999,62 +1003,6 @@ public void testEnqueueTimeThroughLifeCycle() throws Exception {
     checkEnqueueTime(enqueueTime);
   }
 
-  @Test
-  public void testFindPotentialCompactions_limitFetchSize() throws Exception {

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293044386


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java:
##########
@@ -136,7 +136,7 @@ protected Partition resolvePartition(CompactionInfo ci) throws MetaException {
       }
       if (parts.size() != 1) {
         LOG.error(ci.getFullPartitionName() + " does not refer to a single partition. " +
-                      Arrays.toString(parts.toArray()));
+            Arrays.toString(parts.toArray()));

Review Comment:
   please revert this class as there are no changes



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337260260


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/FindPotentialCompactionsFunction.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.TransactionalFunction;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class FindPotentialCompactionsFunction implements TransactionalFunction<Set<CompactionInfo>> {
+  
+  private final int fetchSize;
+  private final int abortedThreshold;
+  private final long abortedTimeThreshold;
+  private final long lastChecked;
+  private final boolean collectAbortedTxns;
+
+  public FindPotentialCompactionsFunction(int fetchSize, int abortedThreshold, long abortedTimeThreshold, long lastChecked, boolean collectAbortedTxns) {
+    this.fetchSize = fetchSize;

Review Comment:
   Shortened the length or arguments, I think now it should be fine without the builder pattern.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/FindPotentialCompactionsFunction.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.TransactionalFunction;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class FindPotentialCompactionsFunction implements TransactionalFunction<Set<CompactionInfo>> {
+  
+  private final int fetchSize;
+  private final int abortedThreshold;
+  private final long abortedTimeThreshold;
+  private final long lastChecked;
+  private final boolean collectAbortedTxns;
+
+  public FindPotentialCompactionsFunction(int fetchSize, int abortedThreshold, long abortedTimeThreshold, long lastChecked, boolean collectAbortedTxns) {
+    this.fetchSize = fetchSize;

Review Comment:
   Shortened the length of the arguments, I think now it should be fine without the builder pattern.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309019954


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1804,263 +572,56 @@ protected void updateWSCommitIdAndCleanUpMetadata(Statement stmt, long txnid, Tx
     }
   }
 
-  private Optional<CompactionInfo> getCompactionByTxnId(Connection dbConn, long txnid) throws SQLException, MetaException {
-    CompactionInfo info = null;
-    try (PreparedStatement pStmt = dbConn.prepareStatement(SELECT_COMPACTION_QUEUE_BY_TXN_ID)) {
-      pStmt.setLong(1, txnid);
-      try (ResultSet rs = pStmt.executeQuery()) {
-        if (rs.next()) {
-          info = CompactionInfo.loadFullFromCompactionQueue(rs);
-        }
-      }
-    }
-    return Optional.ofNullable(info);
-  }
-
   @Override
   public Optional<CompactionInfo> getCompactionByTxnId(long txnId) throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        return getCompactionByTxnId(dbConn, txnId);
-      } catch (SQLException e) {
-        LOG.error("Unable to getCompactionByTxnId", e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "getCompactionByTxnId");
-        throw new MetaException("Unable to execute getCompactionByTxnId() " + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      return getCompactionByTxnId(txnId);
-    }
+    return Optional.ofNullable(getCompactionByTxnId(jdbcTemplate, txnId));
+  }
+
+  private CompactionInfo getCompactionByTxnId(DataSourceWrapper dataSourceWrapper, long txnId) throws MetaException {
+    return dataSourceWrapper.execute( new GetCompactionInfoHandler(txnId, true)); 
   }
 
   @Override
-  protected void createCommitNotificationEvent(Connection dbConn, long txnid, TxnType txnType)
+  protected void createCommitNotificationEvent(DataSourceWrapper dataSourceWrapper, long txnid, TxnType txnType)
       throws MetaException, SQLException {
-    super.createCommitNotificationEvent(dbConn, txnid, txnType);
+    super.createCommitNotificationEvent(dataSourceWrapper, txnid, txnType);
     if (transactionalListeners != null) {
-      Optional<CompactionInfo> compactionInfo = getCompactionByTxnId(dbConn, txnid);
-      if (compactionInfo.isPresent()) {
+      //Please note that TxnHandler and CompactionTxnHandler are using different DataSources (to have different pools).
+      //This call must use the same transaction and connection as TxnHandler.commitTxn(), therefore we are passing the 
+      //datasource wrapper comming from TxnHandler. Without this, the getCompactionByTxnId(long txnId) call would be
+      //executed using a different connection obtained from CompactionTxnHandler's own datasourceWrapper. 
+      CompactionInfo compactionInfo = getCompactionByTxnId(dataSourceWrapper, txnid);
+      if (compactionInfo != null) {
         MetaStoreListenerNotifier
             .notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.COMMIT_COMPACTION,
-                new CommitCompactionEvent(txnid, compactionInfo.get()), dbConn, sqlGenerator);
+                new CommitCompactionEvent(txnid, compactionInfo), dataSourceWrapper.getConnection(), sqlGenerator);
       } else {
         LOG.warn("No compaction queue record found for Compaction type transaction commit. txnId:" + txnid);
       }
     }
   }
-
+  
   @Override
   public boolean updateCompactionMetricsData(CompactionMetricsData data) throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        boolean updateRes;
-        CompactionMetricsData prevMetricsData = getCompactionMetricsData(data, dbConn);
-        if (data.getMetricValue() >= data.getThreshold()) {
-          if (prevMetricsData != null) {
-            updateRes = updateCompactionMetricsData(dbConn, data, prevMetricsData);
-          } else {
-            updateRes = createCompactionMetricsData(dbConn, data);
-          }
-        } else {
-          if (prevMetricsData != null) {
-            updateRes =
-                removeCompactionMetricsData(dbConn, data.getDbName(), data.getTblName(), data.getPartitionName(), data.getMetricType());
-          } else {
-            return true;
-          }
-        }
-        return updateRes;
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "updateCompactionMetricsData(" + data + ")");
-        throw new MetaException("Unable to execute updateCompactionMetricsData()" + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      updateCompactionMetricsData(data);
-    }
-    return false;
+    return new UpdateCompactionMetricsDataFunction(data).execute(jdbcTemplate);

Review Comment:
   jdbcTemplate.execute(UpdateCompactionMetricsDataFunction(data))



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308776504


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +93,12 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
+    Set<CompactionInfo> candidates = new HashSet<>(jdbcTemplate.execute(

Review Comment:
   can we do a set transformation when we return



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308976341


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -444,136 +158,15 @@ public void markCompacted(CompactionInfo info) throws MetaException {
   @Override
   @RetrySemantics.ReadOnly
   public List<CompactionInfo> findReadyToClean(long minOpenTxnWaterMark, long retentionTime) throws MetaException {
-    try {
-      List<CompactionInfo> rc = new ArrayList<>();
-
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-          Statement stmt = dbConn.createStatement()) {
-        /*
-         * By filtering on minOpenTxnWaterMark, we will only cleanup after every transaction is committed, that could see
-         * the uncompacted deltas. This way the cleaner can clean up everything that was made obsolete by this compaction.
-         */
-        String whereClause = " WHERE \"CQ_STATE\" = " + quoteChar(READY_FOR_CLEANING) +
-          " AND \"CQ_TYPE\" != " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " AND (\"CQ_COMMIT_TIME\" < (" + getEpochFn(dbProduct) + " - \"CQ_RETRY_RETENTION\" - " + retentionTime + ") OR \"CQ_COMMIT_TIME\" IS NULL)";
-        
-        String queryStr = 
-          "  \"CQ_ID\", \"cq1\".\"CQ_DATABASE\", \"cq1\".\"CQ_TABLE\", \"cq1\".\"CQ_PARTITION\"," +
-          "  \"CQ_TYPE\", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_TBLPROPERTIES\", \"CQ_RETRY_RETENTION\", " +
-          "  \"CQ_NEXT_TXN_ID\"";
-        if (useMinHistoryWriteId) {
-          queryStr += ", \"MIN_OPEN_WRITE_ID\"";
-        }
-        queryStr +=
-          "  FROM \"COMPACTION_QUEUE\" \"cq1\" " +
-          "INNER JOIN (" +
-          "  SELECT MIN(\"CQ_HIGHEST_WRITE_ID\") \"MIN_WRITE_ID_HWM\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\"" +
-          "  FROM \"COMPACTION_QUEUE\"" 
-          + whereClause +
-          "  GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"cq2\" " +
-          "ON \"cq1\".\"CQ_DATABASE\" = \"cq2\".\"CQ_DATABASE\""+
-          "  AND \"cq1\".\"CQ_TABLE\" = \"cq2\".\"CQ_TABLE\""+
-          "  AND (\"cq1\".\"CQ_PARTITION\" = \"cq2\".\"CQ_PARTITION\"" +
-          "    OR \"cq1\".\"CQ_PARTITION\" IS NULL AND \"cq2\".\"CQ_PARTITION\" IS NULL)" +
-          "  AND \"CQ_HIGHEST_WRITE_ID\" = \"MIN_WRITE_ID_HWM\" ";
-        
-        if (useMinHistoryWriteId) {
-          queryStr += 
-            "LEFT JOIN (" +
-            "  SELECT MIN(\"MH_WRITEID\") \"MIN_OPEN_WRITE_ID\", \"MH_DATABASE\", \"MH_TABLE\"" +
-            "  FROM \"MIN_HISTORY_WRITE_ID\"" +
-            "  GROUP BY \"MH_DATABASE\", \"MH_TABLE\") \"hwm\" " +
-            "ON \"cq1\".\"CQ_DATABASE\" = \"hwm\".\"MH_DATABASE\"" +
-            "  AND \"cq1\".\"CQ_TABLE\" = \"hwm\".\"MH_TABLE\"";
-          
-          whereClause += " AND (\"CQ_HIGHEST_WRITE_ID\" < \"MIN_OPEN_WRITE_ID\" OR \"MIN_OPEN_WRITE_ID\" IS NULL)";
-          
-        } else if (minOpenTxnWaterMark > 0) {
-          whereClause += " AND (\"CQ_NEXT_TXN_ID\" <= " + minOpenTxnWaterMark + " OR \"CQ_NEXT_TXN_ID\" IS NULL)";
-        }
-
-        queryStr += whereClause + " ORDER BY \"CQ_ID\"";
-
-        queryStr = dbProduct.addLimitClause(MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE), queryStr);
-        LOG.debug("Going to execute query <{}>", queryStr);
-
-        try (ResultSet rs = stmt.executeQuery(queryStr)) {
-          while (rs.next()) {
-            CompactionInfo info = new CompactionInfo();
-            info.id = rs.getLong(1);
-            info.dbname = rs.getString(2);
-            info.tableName = rs.getString(3);
-            info.partName = rs.getString(4);
-            info.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(5).charAt(0));
-            info.runAs = rs.getString(6);
-            info.highestWriteId = rs.getLong(7);
-            info.properties = rs.getString(8);
-            info.retryRetention = rs.getInt(9);
-            info.nextTxnId = rs.getLong(10);
-            if (useMinHistoryWriteId) {
-              info.minOpenWriteId = rs.getLong(11);
-            }
-            LOG.debug("Found ready to clean: {}", info);
-            rc.add(info);
-          }
-        }
-        return rc;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
-        checkRetryable(e, "findReadyToClean");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      return findReadyToClean(minOpenTxnWaterMark, retentionTime);
-    }
+    return jdbcTemplate.execute(new ReadyToCleanHandler(useMinHistoryWriteId, minOpenTxnWaterMark, retentionTime, 
+        MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE)));
   }
 
   @Override
   @RetrySemantics.ReadOnly
   public List<CompactionInfo> findReadyToCleanAborts(long abortedTimeThreshold, int abortedThreshold) throws MetaException {
-    try {
-      List<CompactionInfo> readyToCleanAborts = new ArrayList<>();
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-           Statement stmt = dbConn.createStatement()) {
-        boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-
-        String sCheckAborted = dbProduct.addLimitClause(
-                MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-                String.format(SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY,
-                checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold, getEpochFn(dbProduct)));
-        LOG.debug("Going to execute query <{}>", sCheckAborted);
-
-        try (ResultSet rs = stmt.executeQuery(sCheckAborted)) {
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              // In this case, this field contains min open write txn ID.
-              info.minOpenWriteTxnId = rs.getLong(6) > 0 ? rs.getLong(6) : Long.MAX_VALUE;
-              // The specific type, state assigned to abort cleanup.
-              info.type = CompactionType.ABORT_TXN_CLEANUP;
-              info.state = READY_FOR_CLEANING;
-              info.retryRetention = rs.getLong(7);
-              info.id = rs.getLong(8);
-              readyToCleanAborts.add(info);
-            }
-          }
-        }
-        return readyToCleanAborts;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
-        checkRetryable(e, "findReadyToCleanAborts");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      return findReadyToCleanAborts(abortedTimeThreshold, abortedThreshold);
-    }
+    return jdbcTemplate.execute(new AbortTxnInfoHandler(abortedTimeThreshold, abortedThreshold,

Review Comment:
   should it be ReadyToCleanAbortHandler?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309107696


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1603,21 +1645,24 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
         }
 
         if (!isHiveReplTxn) {
-          createCommitNotificationEvent(dbConn, txnid , txnType);
+          createCommitNotificationEvent(jdbcTemplate, txnid , txnType);

Review Comment:
   sh9uld it be dataSourceWrapper.getConnection()?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309963653


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -882,24 +926,15 @@ public void setOpenTxnTimeOutMillis(long openTxnTimeOutMillis) {
     TxnHandler.openTxnTimeOutMillis = openTxnTimeOutMillis;
   }
 
-  protected long getOpenTxnTimeoutLowBoundaryTxnId(Connection dbConn) throws MetaException, SQLException {
-    long maxTxnId;
-    String s =
-        "SELECT MAX(\"TXN_ID\") FROM \"TXNS\" WHERE \"TXN_STARTED\" < (" + getEpochFn(dbProduct) + " - "
-            + openTxnTimeOutMillis + ")";
-    try (Statement stmt = dbConn.createStatement()) {
-      LOG.debug("Going to execute query <{}>", s);
-      try (ResultSet maxTxnIdRs = stmt.executeQuery(s)) {
-        maxTxnIdRs.next();
-        maxTxnId = maxTxnIdRs.getLong(1);
-        if (maxTxnIdRs.wasNull()) {
-          /*
-           * TXNS always contains at least one transaction,
-           * the row where txnid = (select max(txnid) where txn_started < epoch - TXN_OPENTXN_TIMEOUT) is never deleted
-           */
-          throw new MetaException("Transaction tables not properly " + "initialized, null record found in MAX(TXN_ID)");
-        }
-      }
+  protected long getOpenTxnTimeoutLowBoundaryTxnId() throws MetaException {    
+    Long maxTxnId = jdbcTemplate.getJdbcTemplate().queryForObject("SELECT MAX(\"TXN_ID\") FROM \"TXNS\" WHERE \"TXN_STARTED\" < (" + getEpochFn(dbProduct) + " - "

Review Comment:
   JdbcTemplate works the following way: 
   
   - if we are inside a transaction, the associated connection (and transaction) will be used for all jdbctemplate calls. 
   - If we are not in a transaction the call is considered a single transactional call: a connection and trasaction is obtained, then released.
   
   this method called from two places, both initiated a txn before, so we are reusing the connection as before.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309887416


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/AbortTxnInfoHandler.java:
##########
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnStatus;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.QueryHandler;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.hive.metastore.txn.TxnStore.READY_FOR_CLEANING;
+import static org.apache.hadoop.hive.metastore.txn.TxnUtils.getEpochFn;
+
+public class AbortTxnInfoHandler implements QueryHandler<List<CompactionInfo>> {
+
+  // Three inner sub-queries which are under left-join to fetch the required data for aborted txns.
+  //language=SQL
+  private static final String SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY =
+      " \"res1\".\"TC_DATABASE\" AS \"DB\", \"res1\".\"TC_TABLE\" AS \"TBL\", \"res1\".\"TC_PARTITION\" AS \"PART\", " +
+          " \"res1\".\"MIN_TXN_START_TIME\" AS \"MIN_TXN_START_TIME\", \"res1\".\"ABORTED_TXN_COUNT\" AS \"ABORTED_TXN_COUNT\", " +
+          " \"res2\".\"MIN_OPEN_WRITE_TXNID\" AS \"MIN_OPEN_WRITE_TXNID\", \"res3\".\"RETRY_RETENTION\" AS \"RETRY_RETENTION\", " +
+          " \"res3\".\"ID\" AS \"RETRY_CQ_ID\" " +
+          " FROM " +
+          // First sub-query - Gets the aborted txns with min txn start time, number of aborted txns
+          // for corresponding db, table, partition.
+          " ( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TXN_STARTED\") AS \"MIN_TXN_START_TIME\", " +
+          " COUNT(*) AS \"ABORTED_TXN_COUNT\" FROM \"TXNS\", \"TXN_COMPONENTS\" " +
+          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = :abortedState" +
+          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" %s ) \"res1\" " +
+          " LEFT JOIN" +
+          // Second sub-query - Gets the min open txn id for corresponding db, table, partition.
+          "( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TC_TXNID\") AS \"MIN_OPEN_WRITE_TXNID\" " +
+          " FROM \"TXNS\", \"TXN_COMPONENTS\" " +
+          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = :openState" +
+          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" ) \"res2\"" +
+          " ON \"res1\".\"TC_DATABASE\" = \"res2\".\"TC_DATABASE\"" +
+          " AND \"res1\".\"TC_TABLE\" = \"res2\".\"TC_TABLE\"" +
+          " AND (\"res1\".\"TC_PARTITION\" = \"res2\".\"TC_PARTITION\" " +
+          " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res2\".\"TC_PARTITION\" IS NULL)) " +
+          " LEFT JOIN " +
+          // Third sub-query - Gets the retry entries for corresponding db, table, partition.
+          "( SELECT \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", MAX(\"CQ_ID\") AS \"ID\", " +
+          " MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RETENTION\", " +
+          " MIN(\"CQ_COMMIT_TIME\") - %s + MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RECORD_CHECK\" FROM \"COMPACTION_QUEUE\" " +
+          " WHERE \"CQ_TYPE\" = :type" +
+          " GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"res3\" " +
+          " ON \"res1\".\"TC_DATABASE\" = \"res3\".\"CQ_DATABASE\" " +
+          " AND \"res1\".\"TC_TABLE\" = \"res3\".\"CQ_TABLE\" " +
+          " AND (\"res1\".\"TC_PARTITION\" = \"res3\".\"CQ_PARTITION\" " +
+          " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res3\".\"CQ_PARTITION\" IS NULL))" +
+          " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL";
+
+  private final long abortedTimeThreshold;
+  private final int abortedThreshold;
+  private final int fetchSize;
+  
+  public String getParameterizedQueryString(DatabaseProduct dbProduct) throws MetaException {
+    return dbProduct.addLimitClause(
+        fetchSize,
+        String.format(AbortTxnInfoHandler.SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY,
+            abortedTimeThreshold >= 0 ? "" : " HAVING COUNT(*) > " + abortedThreshold, getEpochFn(dbProduct)));
+  }
+
+  @Override
+  public SqlParameterSource getQueryParameters() {
+    return new MapSqlParameterSource()
+        .addValue("abortedState", TxnStatus.ABORTED.getSqlConst(), Types.CHAR)
+        .addValue("openState", TxnStatus.OPEN.getSqlConst(), Types.CHAR)
+        .addValue("type", Character.toString(TxnStore.ABORT_TXN_CLEANUP_TYPE), Types.CHAR);
+  }
+
+  @Override
+  public List<CompactionInfo> extractData(ResultSet rs) throws DataAccessException, SQLException {
+    List<CompactionInfo> readyToCleanAborts = new ArrayList<>();
+    long systemTime = System.currentTimeMillis();
+    boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
+    while (rs.next()) {
+      boolean pastTimeThreshold =
+          checkAbortedTimeThreshold && rs.getLong("MIN_TXN_START_TIME") + abortedTimeThreshold < systemTime;
+      int numAbortedTxns = rs.getInt("ABORTED_TXN_COUNT");
+      if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
+        CompactionInfo info = new CompactionInfo();
+        info.dbname = rs.getString("DB");
+        info.tableName = rs.getString("TBL");
+        info.partName = rs.getString("PART");
+        // In this case, this field contains min open write txn ID.
+        long value = rs.getLong("MIN_OPEN_WRITE_TXNID");
+        info.minOpenWriteTxnId = value > 0 ? value : Long.MAX_VALUE;
+        // The specific type, state assigned to abort cleanup.
+        info.type = CompactionType.ABORT_TXN_CLEANUP;
+        info.state = READY_FOR_CLEANING;
+        info.retryRetention = rs.getLong("RETRY_RETENTION");
+        info.id = rs.getLong("RETRY_CQ_ID");
+        readyToCleanAborts.add(info);

Review Comment:
   Sure, I'll create a task for it.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309969036


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1418,13 +1453,17 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
 
     boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
     boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+    TransactionStatus status = null;

Review Comment:
   Fixed. Not storing the result of `jdbcTemplate.getTransactionWithinRetryContext()` as it is not used explicitly at all.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310072716


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1418,13 +1453,17 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
 
     boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
     boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+    TransactionStatus status = null;
     try {
+      //start a new transaction
+      status = jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);

Review Comment:
   why do we need transactioin with retry context if `checkRetryable(e, "commitTxn(" + rqst + ")");` remained
   that looks very confusing when we have mix of strategies



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1229825415


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state
+    try {

Review Comment:
   The reflection based mapper works only if the field names and column names match, which is quite fragile.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1625087936

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [86 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1613092714

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [66 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1613646185

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1720260136

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1722195789

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337112887


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1047,36 +296,14 @@ public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
   @Override
   @RetrySemantics.Idempotent
   public void revokeFromLocalWorkers(String hostname) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL, \"CQ_STATE\" = '"
-          + INITIATED_STATE+ "' WHERE \"CQ_STATE\" = '" + WORKING_STATE + "' AND \"CQ_WORKER_ID\" LIKE '"
-          +  hostname + "%'";
-        LOG.debug("Going to execute update <{}>", s);
-        // It isn't an error if the following returns no rows, as the local workers could have died
-        // with  nothing assigned to them.
-        int updated = stmt.executeUpdate(s);
-        LOG.debug("Set {} compaction queue entries to {} state for host {}", updated, INITIATED_RESPONSE, hostname);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to change dead worker's records back to initiated state " +
-          e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "revokeFromLocalWorkers(hostname:" + hostname +")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      revokeFromLocalWorkers(hostname);
-    }
+    jdbcResource.execute(
+        "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL," +

Review Comment:
   should we use `IS NULL` instead of `= NULL`



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1047,36 +296,14 @@ public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
   @Override
   @RetrySemantics.Idempotent
   public void revokeFromLocalWorkers(String hostname) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL, \"CQ_STATE\" = '"
-          + INITIATED_STATE+ "' WHERE \"CQ_STATE\" = '" + WORKING_STATE + "' AND \"CQ_WORKER_ID\" LIKE '"
-          +  hostname + "%'";
-        LOG.debug("Going to execute update <{}>", s);
-        // It isn't an error if the following returns no rows, as the local workers could have died
-        // with  nothing assigned to them.
-        int updated = stmt.executeUpdate(s);
-        LOG.debug("Set {} compaction queue entries to {} state for host {}", updated, INITIATED_RESPONSE, hostname);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to change dead worker's records back to initiated state " +
-          e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "revokeFromLocalWorkers(hostname:" + hostname +")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      revokeFromLocalWorkers(hostname);
-    }
+    jdbcResource.execute(
+        "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL," +

Review Comment:
   should we use `IS NULL` instead of `= NULL`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337130708


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/FindPotentialCompactionsFunction.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.TransactionalFunction;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class FindPotentialCompactionsFunction implements TransactionalFunction<Set<CompactionInfo>> {
+  
+  private final int fetchSize;
+  private final int abortedThreshold;
+  private final long abortedTimeThreshold;
+  private final long lastChecked;
+  private final boolean collectAbortedTxns;
+
+  public FindPotentialCompactionsFunction(int fetchSize, int abortedThreshold, long abortedTimeThreshold, long lastChecked, boolean collectAbortedTxns) {
+    this.fetchSize = fetchSize;

Review Comment:
   that is minor, just to consider as list of params in constructor is long



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310142264


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2575,11 +2601,17 @@ protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException,
           minOpenTxn = Long.MAX_VALUE;
         }
       }
+    } catch (SQLException e) {
+      throw new UncategorizedSQLException(null, null, e);
+    }
+    jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);
+    try {
+      long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();

Review Comment:
   Fixed.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1603,21 +1645,24 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
         }
 
         if (!isHiveReplTxn) {
-          createCommitNotificationEvent(dbConn, txnid , txnType);
+          createCommitNotificationEvent(jdbcTemplate, txnid , txnType);
         }
 
         LOG.debug("Going to commit");
-        dbConn.commit();
+        jdbcTemplate.commit();
 
         if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_EXT_ON)) {
           Metrics.getOrCreateCounter(MetricsConstants.TOTAL_NUM_COMMITTED_TXNS).inc();
         }
       } catch (SQLException e) {
         LOG.debug("Going to rollback: ", e);
-        rollbackDBConn(dbConn);
+        jdbcTemplate.rollback();

Review Comment:
   It's also the DataSourceWrapper object, just renamed it to jdbcWrapper according to one of your previos comments.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java:
##########
@@ -79,8 +79,8 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
   public Set<Long> writeIds;
   public boolean hasUncompactedAborts;
 
-  byte[] metaInfo;
-  String hadoopJobId;
+  public byte[] metaInfo;
+  public String hadoopJobId;

Review Comment:
   I think not, with @deniskuzZ  we discussed that CompactionInfo should not be touched as a part of this PR.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5755,80 +5807,86 @@ private void timeOutLocks(Connection dbConn) {
    */
   @RetrySemantics.Idempotent
   public void performTimeOuts() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
     try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-      //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
-      //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
-      //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
-      //would be to make sure someone cannot heartbeat one of these txns at the same time.
-      //The attempt to heartbeat would block and fail immediately after it's unblocked.
-      //With current (RC + multiple txns) implementation it is possible for someone to send
-      //heartbeat at the very end of the expire interval, and just after the Select from TXNS
-      //is made, in which case heartbeat will succeed but txn will still be Aborted.
-      //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
-      //timely way.
-      timeOutLocks(dbConn);
-      while(true) {
-        stmt = dbConn.createStatement();
-        String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " AND (" +
-                "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
-             " OR " +
-                " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
-             ")";
-        //safety valve for extreme cases
-        s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if(!rs.next()) {
-          return;//no more timedout txns
-        }
-        List<List<Long>> timedOutTxns = new ArrayList<>();
-        List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-        timedOutTxns.add(currentBatch);
-        do {
-          if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
-            currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-            timedOutTxns.add(currentBatch);
-          }
-          currentBatch.add(rs.getLong(1));
-        } while(rs.next());
-        dbConn.commit();
-        close(rs, stmt, null);
-        int numTxnsAborted = 0;
-        for(List<Long> batchToAbort : timedOutTxns) {
-          if (abortTxns(dbConn, batchToAbort, true, false, false, TxnErrorMsg.ABORT_TIMEOUT) == batchToAbort.size()) {
-            dbConn.commit();
-            numTxnsAborted += batchToAbort.size();
-            //todo: add TXNS.COMMENT filed and set it to 'aborted by system due to timeout'
-          }
-          else {
-            //could not abort all txns in this batch - this may happen because in parallel with this
-            //operation there was activity on one of the txns in this batch (commit/abort/heartbeat)
-            //This is not likely but may happen if client experiences long pause between heartbeats or
-            //unusually long/extreme pauses between heartbeat() calls and other logic in checkLock(),
-            //lock(), etc.
-            dbConn.rollback();
+      retryHandler.executeWithoutRetry(
+          new RetryCallProperties()
+              .withCallerId("performTimeOuts()")
+              .withDataSource(POOL_TX)
+              .withExceptionSupplier(e -> new MetaException("Aborting timed out transactions failed due to " + RetryHandler.getMessage(e))),
+          (DataSourceWrapper dataSourceWrapper) -> {

Review Comment:
   done



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1418,13 +1453,17 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
 
     boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
     boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+    TransactionStatus status = null;
     try {
+      //start a new transaction
+      status = jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);

Review Comment:
   You are absolutely right on this one. The reason for this mixing approach is the following: To reduce the size of the PR (it's already huge) I split it to CompactionTxnHandler and TxnHandler cleanup. Unfortunately there are situations where a TxnHandler method calls a CompactionTxnHandler one or vice versa. In these cases there were three options:
   
   - Skip these methods from the redesign for now. I dropped this aproach as it would lead to leaving almost half of CompactionTxnHandler untouched. So those changes would make the second PR larger.
   - Redesign the necessary TxnHandler methods as well. I dropped this approach as it would lead to redesigning at least 1/3 of TxnHandler in this PR.
   - My chosen approach: In the affected TxnHandler methods replace the getConnection and commit/rollback calls only. As a result the RetryContext is established and CompcationTxnHandler methods can be called without passing the Connection object. On the other hand, changes in TxnHandler are very limited, the rest of these methods are untouched.
   
   Of course this mixed approach is bad and therefore must be temporary. However, I still thought this is the best compromise from the possibilities. I already started removing them in the TxnHandler cleanup PR.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5755,80 +5807,86 @@ private void timeOutLocks(Connection dbConn) {
    */
   @RetrySemantics.Idempotent
   public void performTimeOuts() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
     try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-      //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
-      //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
-      //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
-      //would be to make sure someone cannot heartbeat one of these txns at the same time.
-      //The attempt to heartbeat would block and fail immediately after it's unblocked.
-      //With current (RC + multiple txns) implementation it is possible for someone to send
-      //heartbeat at the very end of the expire interval, and just after the Select from TXNS
-      //is made, in which case heartbeat will succeed but txn will still be Aborted.
-      //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
-      //timely way.
-      timeOutLocks(dbConn);
-      while(true) {
-        stmt = dbConn.createStatement();
-        String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " AND (" +
-                "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
-             " OR " +
-                " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
-             ")";
-        //safety valve for extreme cases
-        s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if(!rs.next()) {
-          return;//no more timedout txns
-        }
-        List<List<Long>> timedOutTxns = new ArrayList<>();
-        List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-        timedOutTxns.add(currentBatch);
-        do {
-          if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
-            currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-            timedOutTxns.add(currentBatch);
-          }
-          currentBatch.add(rs.getLong(1));
-        } while(rs.next());
-        dbConn.commit();
-        close(rs, stmt, null);
-        int numTxnsAborted = 0;
-        for(List<Long> batchToAbort : timedOutTxns) {
-          if (abortTxns(dbConn, batchToAbort, true, false, false, TxnErrorMsg.ABORT_TIMEOUT) == batchToAbort.size()) {
-            dbConn.commit();
-            numTxnsAborted += batchToAbort.size();
-            //todo: add TXNS.COMMENT filed and set it to 'aborted by system due to timeout'
-          }
-          else {
-            //could not abort all txns in this batch - this may happen because in parallel with this
-            //operation there was activity on one of the txns in this batch (commit/abort/heartbeat)
-            //This is not likely but may happen if client experiences long pause between heartbeats or
-            //unusually long/extreme pauses between heartbeat() calls and other logic in checkLock(),
-            //lock(), etc.
-            dbConn.rollback();
+      retryHandler.executeWithoutRetry(
+          new RetryCallProperties()
+              .withCallerId("performTimeOuts()")
+              .withDataSource(POOL_TX)
+              .withExceptionSupplier(e -> new MetaException("Aborting timed out transactions failed due to " + RetryHandler.getMessage(e))),
+          (DataSourceWrapper dataSourceWrapper) -> {
+            //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
+            //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
+            //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
+            //would be to make sure someone cannot heartbeat one of these txns at the same time.
+            //The attempt to heartbeat would block and fail immediately after it's unblocked.
+            //With current (RC + multiple txns) implementation it is possible for someone to send
+            //heartbeat at the very end of the expiry interval, and just after the Select from TXNS
+            //is made, in which case heartbeat will succeed but txn will still be Aborted.
+            //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
+            //timely way.
+            timeOutLocks();
+            while (true) {
+              String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
+                  " AND (" +
+                  "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
+                  " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
+                  " OR " +
+                  " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
+                  " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
+                  ")";
+              //safety valve for extreme cases
+              s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
+
+              LOG.debug("Going to execute query <{}>", s);
+              List<List<Long>> timedOutTxns = dataSourceWrapper.getJdbcTemplate().query(s, rs -> {
+                List<List<Long>> txnbatch = new ArrayList<>();
+                List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+                while (rs.next()) {
+                  currentBatch.add(rs.getLong(1));
+                  if (currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
+                    txnbatch.add(currentBatch);
+                    currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+                  }
+                }
+                if (currentBatch.size() > 0) {
+                  txnbatch.add(currentBatch);
+                }
+                return txnbatch;
+              });
+              //noinspection DataFlowIssue
+              if (timedOutTxns.size() == 0) {
+                return Void.TYPE;
+              }
+
+              TransactionStatus status = dataSourceWrapper.getTransactionStatus();
+              Object savePoint = status.createSavepoint();
+
+              int numTxnsAborted = 0;
+              for (List<Long> batchToAbort : timedOutTxns) {
+                status.releaseSavepoint(savePoint);
+                savePoint = status.createSavepoint();

Review Comment:
   This is just the jdbcTemplate equivalent of the old code, it had commit/rollback calls inside the loop. We can enhance it but I would do it in a separate PR.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1285833195


##########
service/pom.xml:
##########
@@ -221,6 +221,10 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.springframework</groupId>
+      <artifactId>spring-core</artifactId>

Review Comment:
   Spring core is required for exception handling. Spring converts SqlException to DataAccessException (which is runtime not checked). During the conversion, based on the SQL codes for some exceptions (but not for all) spring also localizes the exception message. This is the point where spring-core is required. It is not used directly at all, no spring DI introduced in this PR



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293046456


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/AbortedTxnCleaner.java:
##########
@@ -102,7 +102,7 @@ private void clean(CompactionInfo info, long minOpenWriteTxn, boolean metricsEna
       if (isNull(t)) {
         // The table was dropped before we got around to cleaning it.
         LOG.info("Unable to find table {}, assuming it was dropped.", info.getFullTableName());
-        txnHandler.markCleaned(info);
+        txnHandler.markCleaned((CompactionInfo) info);

Review Comment:
   redundant case, please revert the changes



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293050405


##########
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java:
##########
@@ -999,62 +1003,6 @@ public void testEnqueueTimeThroughLifeCycle() throws Exception {
     checkEnqueueTime(enqueueTime);
   }
 
-  @Test
-  public void testFindPotentialCompactions_limitFetchSize() throws Exception {

Review Comment:
   why remove these tests? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293133920


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,38 +261,22 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction)) {
-        try (PreparedStatement stmt = dbConn.prepareStatement(DELETE_FAILED_TXNS_SQL)) {
-          //Aborted and committed are terminal states, so nothing about the txn can change
-          //after that, so READ COMMITTED is sufficient.
-          /*
-           * Only delete aborted / committed transaction in a way that guarantees two things:
-           * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
-           * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
-           */
-          long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId(dbConn);
-          stmt.setLong(1, lowWaterMark);
-          LOG.debug("Going to execute query <{}>", DELETE_FAILED_TXNS_SQL);
-          int rc = stmt.executeUpdate();
-          LOG.debug("Removed {} empty Aborted and Committed transactions from TXNS", rc);
-          LOG.debug("Going to commit");
-          dbConn.commit();
-        } catch (SQLException e) {
-          LOG.error("Unable to delete from txns table " + e.getMessage());
-          LOG.debug("Going to rollback");
-          rollbackDBConn(dbConn);
-          checkRetryable(e, "cleanEmptyAbortedTxns");
-          throw new MetaException("Unable to delete from txns table " + e.getMessage());
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e, "cleanEmptyAbortedTxns");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedAndCommittedTxns();
-    }
+    //Aborted and committed are terminal states, so nothing about the txn can change
+    //after that, so READ COMMITTED is sufficient.
+    /*
+     * Only delete aborted / committed transaction in a way that guarantees two things:
+     * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
+     * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
+     */
+    long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+    String query = "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +

Review Comment:
   I tried to keep the queries inside the QueryHandler or inside the TxnHandler method if there is no separate QueryHandler implementation. Te reason is simply to keep the query as close as possible to the code where it is used. I think it improves readability. However, I'm not strictly tied to it. Should I revert this part and use constants instead?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293081977


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +141,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    ParameterizedCommand.execute(dataSourceWrapper, 

Review Comment:
   Originally this stlye was used everywhere, I have no strong feelings about it, I can restore it in QueryHandler also.
   I can embed the execute methods in `DataSourceWrapper`, so it would look like as you wrote but `jdbcTemplate` would be `datasourceWrapper`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293133920


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,38 +261,22 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction)) {
-        try (PreparedStatement stmt = dbConn.prepareStatement(DELETE_FAILED_TXNS_SQL)) {
-          //Aborted and committed are terminal states, so nothing about the txn can change
-          //after that, so READ COMMITTED is sufficient.
-          /*
-           * Only delete aborted / committed transaction in a way that guarantees two things:
-           * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
-           * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
-           */
-          long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId(dbConn);
-          stmt.setLong(1, lowWaterMark);
-          LOG.debug("Going to execute query <{}>", DELETE_FAILED_TXNS_SQL);
-          int rc = stmt.executeUpdate();
-          LOG.debug("Removed {} empty Aborted and Committed transactions from TXNS", rc);
-          LOG.debug("Going to commit");
-          dbConn.commit();
-        } catch (SQLException e) {
-          LOG.error("Unable to delete from txns table " + e.getMessage());
-          LOG.debug("Going to rollback");
-          rollbackDBConn(dbConn);
-          checkRetryable(e, "cleanEmptyAbortedTxns");
-          throw new MetaException("Unable to delete from txns table " + e.getMessage());
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e, "cleanEmptyAbortedTxns");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedAndCommittedTxns();
-    }
+    //Aborted and committed are terminal states, so nothing about the txn can change
+    //after that, so READ COMMITTED is sufficient.
+    /*
+     * Only delete aborted / committed transaction in a way that guarantees two things:
+     * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
+     * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
+     */
+    long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+    String query = "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +

Review Comment:
   I tried to keep the queries inside the QueryHandler or inside the TxnHandler method if there is no separate QueryHandler implementation. Te reason is simply to keep the query as close as possible to the code where it is used. I think it improves readability. However, I'm not strictly tied to it. Should I revert this part and constants instead?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1574612359

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [32 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309110710


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2502,40 +2546,22 @@ public void seedTxnId(SeedTxnIdRequest rqst) throws MetaException {
 
   @Override
   @RetrySemantics.Idempotent
-  public void addWriteNotificationLog(ListenerEvent acidWriteEvent)
-          throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        //Idempotent case is handled by notify Event
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners,
-                acidWriteEvent instanceof AcidWriteEvent ? EventMessage.EventType.ACID_WRITE
-                        : EventMessage.EventType.BATCH_ACID_WRITE,
-                acidWriteEvent, dbConn, sqlGenerator);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback: ", e);
-        rollbackDBConn(dbConn);
-        if (isDuplicateKeyError(e)) {
-          // in case of key duplicate error, retry as it might be because of race condition
-          if (waitForRetry("addWriteNotificationLog(" + acidWriteEvent + ")", e.getMessage())) {
-            throw new RetryException();
-          }
-          retryNum = 0;
-          throw new MetaException(e.getMessage());
+  public void addWriteNotificationLog(ListenerEvent acidWriteEvent) throws MetaException {
+    retryHandler.executeWithRetry(new RetryCallProperties()

Review Comment:
   can we extract that into some function as looks like it could be reused in multiple places



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309810514


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java:
##########
@@ -405,6 +410,50 @@ public static List<Integer> buildQueryWithINClauseStrings(Configuration conf, Li
     return ret;
   }
 
+  /**
+   * Executes the statement with an IN clause. If the number of elements or the length of the constructed statement would be
+   * too big, the IN clause will be split into multiple smaller ranges, and the statement will be executed multiple times.
+   * @param conf Hive configuration used to get the query and IN clause length limits.
+   * @param jdbcTemplate The {@link NamedParameterJdbcTemplate} instance to used for statement execution.
+   * @param query The query with the IN clause
+   * @param params A {@link MapSqlParameterSource} instance with the parameters of the query
+   * @param inClauseParamName The name of the parameter representing the content of the IN clause 
+   * @param elements A {@link List} containing the elements to put in the IN clause
+   * @param comparator A {@link Comparator} instance used to find the longest element in the list. Used to
+   *                   estimate the length of the query.
+   * @return Returns the total number of affected rows.
+   * @param <T> Type of the elements in the list.
+   */
+  public static <T> int executeStatementWithInClause(Configuration conf, NamedParameterJdbcTemplate jdbcTemplate, 

Review Comment:
   shouldn't this be part of util class?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309944351


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -438,6 +467,9 @@ public void setConf(Configuration conf){
       LOG.error(msg);
       throw new RuntimeException(e);
     }
+
+    retryHandler = new RetryHandler();
+    retryHandler.init(conf, jdbcTemplate);    

Review Comment:
   done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309999458


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;
+  }
+
+  /**
+   * @return Returns true if the current {@link Thread} has an associated {@link RetryContext}, false otherwise.
+   */
+  boolean hasRetryContext() {
+    return threadLocal.get() != null;
+  }
+
+  /**
+   * @return Returns the {@link NamedParameterJdbcTemplate} associated with the current {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public NamedParameterJdbcTemplate getJdbcTemplate() {
+    return jdbcTemplates.get(RetryContext.getRetryContext().datasource);
+  }
+
+  /**
+   * Gets a connection to the {@link DataSource} associated with the current {@link RetryContext}. Ensures that the same 
+   * instance is returned all the time within a particular transaction. Can be called only within an established 
+   * {@link RetryContext}. 
+   * @return Returns a connection to the datasource.
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public Connection getConnection() {
+    return DataSourceUtils.getConnection(dataSources.get(RetryContext.getRetryContext().datasource));
+  }
+
+  /**
+   * @return Returns a {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DatabaseProduct getDatabaseProduct() {
+    return databaseProduct;
+  }
+
+  /**
+   * <p>Establishes a {@link RetryContext} on the current {@link Thread}: Creates a new transaction with the given 
+   * {@link TransactionDefinition} using an internal {@link PlatformTransactionManager} associated with the reffered 
+   * {@link DataSource}. Returns a {@link TransactionStatus} representing the created transaction. The returned 
+   * {@link TransactionStatus} instance can be used for manual transaction handling.</p> 
+   * <br>
+   * <p>!!!! Please note that if there already is a {@link RetryContext} established, this call will create a nested</p>
+   * {@link RetryContext} (and transaction).</b>  
+   * @param definition The {@link TransactionDefinition} to use for creating a new transaction.
+   * @param dataSource The identifier of the {@link DataSource} to use
+   * @return Returns a {@link TransactionStatus} object representing the created transaction.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#getTransaction(TransactionDefinition)}
+   */
+  public TransactionStatus getTransactionWithinRetryContext(TransactionDefinition definition, String dataSource) throws TransactionException {
+    TransactionStatus status = transactionManagers.get(dataSource).getTransaction(definition);
+    RetryContext.setRetryContext(status, dataSource);
+    return status;
+  }
+
+  /**
+   * Commits the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#commit(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void commit() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();
+      transactionManagers.get(context.datasource).commit(context.transactionStatus);
+    } finally {
+      RetryContext.clearRetryContext();
+    }
+  }
+
+  /**
+   * Rollbacks the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#rollback(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void rollback() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();

Review Comment:
   In case of nested context, the inner context will be rolled back, while the outer remains untouched.
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309990206


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryCallProperties.java:
##########
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.datanucleus.transaction.TransactionIsolation;
+
+import java.util.function.Function;
+
+/**
+ * Contains all the properties which can alter the behavior of a retry-call in 
+ * {@link org.apache.hadoop.hive.metastore.txn.retryhandling.RetryHandler}.
+ */
+public class RetryCallProperties {
+
+  private String dataSource;
+  private String caller = null;
+  private boolean retryOnDuplicateKey = false;
+  private int transactionIsolationLevel = TransactionIsolation.READ_COMMITTED;

Review Comment:
   Added propagation level too. I think better to have control over these, even if currently all methods using the default value.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310044406


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2502,40 +2546,22 @@ public void seedTxnId(SeedTxnIdRequest rqst) throws MetaException {
 
   @Override
   @RetrySemantics.Idempotent
-  public void addWriteNotificationLog(ListenerEvent acidWriteEvent)
-          throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        //Idempotent case is handled by notify Event
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        MetaStoreListenerNotifier.notifyEventWithDirectSql(transactionalListeners,
-                acidWriteEvent instanceof AcidWriteEvent ? EventMessage.EventType.ACID_WRITE
-                        : EventMessage.EventType.BATCH_ACID_WRITE,
-                acidWriteEvent, dbConn, sqlGenerator);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback: ", e);
-        rollbackDBConn(dbConn);
-        if (isDuplicateKeyError(e)) {
-          // in case of key duplicate error, retry as it might be because of race condition
-          if (waitForRetry("addWriteNotificationLog(" + acidWriteEvent + ")", e.getMessage())) {
-            throw new RetryException();
-          }
-          retryNum = 0;
-          throw new MetaException(e.getMessage());
+  public void addWriteNotificationLog(ListenerEvent acidWriteEvent) throws MetaException {
+    retryHandler.executeWithRetry(new RetryCallProperties()

Review Comment:
   Fixed. Added the `@Retry` annotation and removed the explicit retryHandler call. Before I got the idea of creating the annotation and the `RetryingTxnHandler` proxy, this was the way of using the `RetryHandler`. It's a remnant of the old approach.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310031152


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6397,51 +6458,44 @@ private AbortCompactionResponseElement getAbortCompactionResponseElement(long co
 
   @RetrySemantics.SafeToRetry
   public AbortCompactionResponseElement abortCompaction(CompactionInfo compactionInfo) throws MetaException {
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolMutex);
-           PreparedStatement pStmt = dbConn.prepareStatement(TxnQueries.INSERT_INTO_COMPLETED_COMPACTION)) {
-        compactionInfo.state = TxnStore.ABORTED_STATE;
-        compactionInfo.errorMessage = "Comapction Aborted by Abort Comapction request.";
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, compactionInfo, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        if (updCount != 1) {
-          LOG.error("Unable to update compaction record: {}. updCnt={}", compactionInfo, updCount);
-          dbConn.rollback();
-          return getAbortCompactionResponseElement(compactionInfo.id, "Error",
-                  "Error while aborting compaction:Unable to update compaction record in COMPLETED_COMPACTIONS");
-        } else {
+    return retryHandler.executeWithRetry(new RetryCallProperties()
+            .withCallerId("abortCompaction(" + compactionInfo + ")")
+            .withDataSource(POOL_TX),
+        (DataSourceWrapper dataSourceWrapper) -> {
+          compactionInfo.state = TxnStore.ABORTED_STATE;
+          compactionInfo.errorMessage = "Comapction Aborted by Abort Comapction request.";

Review Comment:
   typo: Compaction



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309019954


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1804,263 +572,56 @@ protected void updateWSCommitIdAndCleanUpMetadata(Statement stmt, long txnid, Tx
     }
   }
 
-  private Optional<CompactionInfo> getCompactionByTxnId(Connection dbConn, long txnid) throws SQLException, MetaException {
-    CompactionInfo info = null;
-    try (PreparedStatement pStmt = dbConn.prepareStatement(SELECT_COMPACTION_QUEUE_BY_TXN_ID)) {
-      pStmt.setLong(1, txnid);
-      try (ResultSet rs = pStmt.executeQuery()) {
-        if (rs.next()) {
-          info = CompactionInfo.loadFullFromCompactionQueue(rs);
-        }
-      }
-    }
-    return Optional.ofNullable(info);
-  }
-
   @Override
   public Optional<CompactionInfo> getCompactionByTxnId(long txnId) throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        return getCompactionByTxnId(dbConn, txnId);
-      } catch (SQLException e) {
-        LOG.error("Unable to getCompactionByTxnId", e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "getCompactionByTxnId");
-        throw new MetaException("Unable to execute getCompactionByTxnId() " + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      return getCompactionByTxnId(txnId);
-    }
+    return Optional.ofNullable(getCompactionByTxnId(jdbcTemplate, txnId));
+  }
+
+  private CompactionInfo getCompactionByTxnId(DataSourceWrapper dataSourceWrapper, long txnId) throws MetaException {
+    return dataSourceWrapper.execute( new GetCompactionInfoHandler(txnId, true)); 
   }
 
   @Override
-  protected void createCommitNotificationEvent(Connection dbConn, long txnid, TxnType txnType)
+  protected void createCommitNotificationEvent(DataSourceWrapper dataSourceWrapper, long txnid, TxnType txnType)
       throws MetaException, SQLException {
-    super.createCommitNotificationEvent(dbConn, txnid, txnType);
+    super.createCommitNotificationEvent(dataSourceWrapper, txnid, txnType);
     if (transactionalListeners != null) {
-      Optional<CompactionInfo> compactionInfo = getCompactionByTxnId(dbConn, txnid);
-      if (compactionInfo.isPresent()) {
+      //Please note that TxnHandler and CompactionTxnHandler are using different DataSources (to have different pools).
+      //This call must use the same transaction and connection as TxnHandler.commitTxn(), therefore we are passing the 
+      //datasource wrapper comming from TxnHandler. Without this, the getCompactionByTxnId(long txnId) call would be
+      //executed using a different connection obtained from CompactionTxnHandler's own datasourceWrapper. 
+      CompactionInfo compactionInfo = getCompactionByTxnId(dataSourceWrapper, txnid);
+      if (compactionInfo != null) {
         MetaStoreListenerNotifier
             .notifyEventWithDirectSql(transactionalListeners, EventMessage.EventType.COMMIT_COMPACTION,
-                new CommitCompactionEvent(txnid, compactionInfo.get()), dbConn, sqlGenerator);
+                new CommitCompactionEvent(txnid, compactionInfo), dataSourceWrapper.getConnection(), sqlGenerator);
       } else {
         LOG.warn("No compaction queue record found for Compaction type transaction commit. txnId:" + txnid);
       }
     }
   }
-
+  
   @Override
   public boolean updateCompactionMetricsData(CompactionMetricsData data) throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        boolean updateRes;
-        CompactionMetricsData prevMetricsData = getCompactionMetricsData(data, dbConn);
-        if (data.getMetricValue() >= data.getThreshold()) {
-          if (prevMetricsData != null) {
-            updateRes = updateCompactionMetricsData(dbConn, data, prevMetricsData);
-          } else {
-            updateRes = createCompactionMetricsData(dbConn, data);
-          }
-        } else {
-          if (prevMetricsData != null) {
-            updateRes =
-                removeCompactionMetricsData(dbConn, data.getDbName(), data.getTblName(), data.getPartitionName(), data.getMetricType());
-          } else {
-            return true;
-          }
-        }
-        return updateRes;
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "updateCompactionMetricsData(" + data + ")");
-        throw new MetaException("Unable to execute updateCompactionMetricsData()" + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      updateCompactionMetricsData(data);
-    }
-    return false;
+    return new UpdateCompactionMetricsDataFunction(data).execute(jdbcTemplate);

Review Comment:
   jdbcTemplate.execute(UpdateCompactionMetricsDataFunction(data))



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309105560


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1418,13 +1453,17 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
 
     boolean isReplayedReplTxn = TxnType.REPL_CREATED.equals(rqst.getTxn_type());
     boolean isHiveReplTxn = rqst.isSetReplPolicy() && TxnType.DEFAULT.equals(rqst.getTxn_type());
+    TransactionStatus status = null;

Review Comment:
   where is it being used?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309775526


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;

Review Comment:
   shouldn't this be` threadLocal.get().transactionStatus`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309025882


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/RetryingTxnHandler.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryCallProperties;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.DataSourceWrapper;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.Retry;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Arrays;
+
+/**
+ * Responsible to use the {@link RetryHandler} to automatically retry the failed {@link TxnStore} methods if annotated with
+ * the {@link Retry} annotation.
+ */
+public class RetryingTxnHandler implements InvocationHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryingTxnHandler.class);
+
+  /**
+   * Gets the proxy interface for the given {@link TxnStore}.
+   * @param realStore The real {@link TxnStore} to proxy.
+   * @param retryHandler Responsible to re-execute the methods in case of failure.
+   * @return Returns the proxy object capable of retrying the failed calls automatically and transparently. 
+   */
+  public static TxnStore getProxy(TxnStore realStore, RetryHandler retryHandler) {
+    RetryingTxnHandler handler = new RetryingTxnHandler(realStore, retryHandler);
+    return (TxnStore) Proxy.newProxyInstance(
+        RetryingTxnHandler.class.getClassLoader(),
+        new Class[]{ TxnStore.class },
+        handler);
+  }
+
+  private final RetryHandler retryHandler;
+  private final TxnStore realStore;
+
+  private RetryingTxnHandler(TxnStore realStore, RetryHandler retryHandler) {
+    this.realStore = realStore;
+    this.retryHandler = retryHandler;
+  }
+
+  @Override
+  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    String strArgs = args == null ? "" : StringUtils.join(Arrays.asList(args), ",");
+    String callerId = method.getName() + "(" + strArgs + ")";
+    Retry retry = method.getAnnotation(Retry.class);
+    if (retry != null) {
+      RetryCallProperties properties = new RetryCallProperties()
+          .withCallerId(callerId)
+          .withDataSource(retry.datasource())
+          .withLockInternally(retry.lockInternally())
+          .withRollbackOnError(retry.rollbackOnError())
+          .withRetryOnDuplicateKey(retry.retryOnDuplicateKey())
+          .withTransactionIsolationLevel(retry.transactionIsolation());
+      return retryHandler.executeWithRetry(properties,
+          (DataSourceWrapper dataSourceWrapper) -> {
+            try {
+              LOG.info("Invoking method within retry context: {}", callerId);

Review Comment:
   should this be under info level?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1226907857


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/SqlParamSourceMapper.java:
##########
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+public interface SqlParamSourceMapper<T> {

Review Comment:
   It's the counterpart of the `RowMapper` interface, map an entity to `SqlParamSource`. Entity mappers can implement one of them or both. Currently it is not in use, but can be handy in the entity mappers need to be handled together for any reason. 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1226907857


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/SqlParamSourceMapper.java:
##########
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+public interface SqlParamSourceMapper<T> {

Review Comment:
   It's the counterpart of the `RowMapper` interface, map an entity to `SqlParamSource`. Entity mappers can implement one of them or both. Currently it is not in use, but can be handy if the entity mappers need to be handled together for any reason. 



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {

Review Comment:
   It's a builder style setter.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1601097165

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [59 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1222919970


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/AbortedTxnCleaner.java:
##########
@@ -99,7 +100,7 @@ private void clean(CompactionInfo info, long minOpenWriteTxn, boolean metricsEna
       if (isNull(t)) {
         // The table was dropped before we got around to cleaning it.
         LOG.info("Unable to find table {}, assuming it was dropped.", info.getFullTableName());
-        txnHandler.markCleaned(info, true);
+        txnHandler.markCleaned((CompactionInfo) info, true);

Review Comment:
   I'm not sure how Thrift will behave when old client interacts with new Server, or vice versa. Changing thrift signatures should be done in a separate PR as it can trigger further changes.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1285848071


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfoBase.java:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+public class CompactionInfoBase implements Comparable<CompactionInfoBase> {

Review Comment:
   Currently CompactionInfo is like an union of all compaction related information. In numerous cases only a fraction of the fields are loaded to save resources (`findNextToCompact`, `findReadyToClean`), and in some other cases like `findPotentialCompactions` in Initiator, several fields are not even applicable for the scenario. This results in a situation that Compaction info is almost never fully initialized and you have to be aware of the context where it is used: You need to know which fields are present and which not. While technically this is a fully working approach, I would prefer introduce some class hierarchy. For example `CompactionCandidate` is the entity used by Initiator. Having this object makes it much more explicit which information is used/available in the initialization phase, and which others are not yet present/applicable. This is the idea behind the introduction of the new base classes.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288089145


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionCandidate.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+
+public class CompactionCandidate extends CompactionInfoBase {
+
+  public boolean tooManyAborts;
+  public boolean hasOldAbort;
+  
+  public CompactionInfo toFailedCompacion(String errorMessage) {

Review Comment:
   why do you need to create a clone of CompactionInfo, instead of just setting the error message?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1671326660

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [76 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309098108


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2575,11 +2601,17 @@ protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException,
           minOpenTxn = Long.MAX_VALUE;
         }
       }
+    } catch (SQLException e) {
+      throw new UncategorizedSQLException(null, null, e);
+    }
+    jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);
+    try {
+      long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+      LOG.debug("MinOpenTxnIdWaterMark calculated with minOpenTxn {}, lowWaterMark {}", minOpenTxn, lowWaterMark);
+      return Long.min(minOpenTxn, lowWaterMark + 1);
+    } finally {
+      jdbcTemplate.commit();

Review Comment:
   would it commit the txn? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308773354


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -58,114 +68,18 @@
  * methods are not available through the thrift interface.
  */
 class CompactionTxnHandler extends TxnHandler {
-  private static final String CLASS_NAME = CompactionTxnHandler.class.getName();
-  private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
-  private static final String DB_FAILED_TO_CONNECT = "Unable to connect to transaction database: ";
-
-  private static DataSource connPoolCompaction;
-
-  private static final String SELECT_COMPACTION_QUEUE_BY_TXN_ID =
-      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-          + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-          + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-          + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-          + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-          + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" "
-          + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?";
-  private static final String SELECT_COMPACTION_METRICS_CACHE_QUERY =
-      "SELECT \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM \"COMPACTION_METRICS_CACHE\" " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ?";
-  private static final String NO_SELECT_COMPACTION_METRICS_CACHE_FOR_TYPE_QUERY =
-      "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM " +
-      "\"COMPACTION_METRICS_CACHE\" WHERE \"CMC_METRIC_TYPE\" = ? ORDER BY \"CMC_METRIC_VALUE\" DESC";
-  private static final String UPDATE_COMPACTION_METRICS_CACHE_QUERY =
-      "UPDATE \"COMPACTION_METRICS_CACHE\" SET \"CMC_METRIC_VALUE\" = ?, \"CMC_VERSION\" = ? " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ? " +
-          "AND \"CMC_VERSION\" = ?";
-  private static final String INSERT_COMPACTION_METRICS_CACHE_QUERY =
-      "INSERT INTO \"COMPACTION_METRICS_CACHE\" ( " +
-          "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_TYPE\", \"CMC_METRIC_VALUE\", " +
-          "\"CMC_VERSION\" ) VALUES (?, ?, ?, ?, ?, ?)";
-  private static final String DELETE_COMPACTION_METRICS_CACHE_QUERY =
-      "DELETE FROM \"COMPACTION_METRICS_CACHE\" WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? " +
-          "AND \"CMC_METRIC_TYPE\" = ?";
-
-  private static final String DELETE_FAILED_TXNS_SQL =
-      "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
-          "AND (\"TXN_STATE\" = " + TxnStatus.ABORTED + " OR \"TXN_STATE\" = " + TxnStatus.COMMITTED + ") " +
-          "AND \"TXN_ID\" < ?";
-
-  // Three inner sub-queries which are under left-join to fetch the required data for aborted txns.
-  private static final String SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY =
-          " \"res1\".\"TC_DATABASE\" AS \"DB\", \"res1\".\"TC_TABLE\" AS \"TBL\", \"res1\".\"TC_PARTITION\" AS \"PART\", " +
-          " \"res1\".\"MIN_TXN_START_TIME\" AS \"MIN_TXN_START_TIME\", \"res1\".\"ABORTED_TXN_COUNT\" AS \"ABORTED_TXN_COUNT\", " +
-          " \"res2\".\"MIN_OPEN_WRITE_TXNID\" AS \"MIN_OPEN_WRITE_TXNID\", \"res3\".\"RETRY_RETENTION\" AS \"RETRY_RETENTION\", " +
-          " \"res3\".\"ID\" AS \"RETRY_CQ_ID\" " +
-      " FROM " +
-          // First sub-query - Gets the aborted txns with min txn start time, number of aborted txns
-          // for corresponding db, table, partition.
-          " ( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TXN_STARTED\") AS \"MIN_TXN_START_TIME\", " +
-          " COUNT(*) AS \"ABORTED_TXN_COUNT\" FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" %s ) \"res1\" " +
-      " LEFT JOIN" +
-          // Second sub-query - Gets the min open txn id for corresponding db, table, partition.
-          "( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TC_TXNID\") AS \"MIN_OPEN_WRITE_TXNID\" " +
-          " FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" ) \"res2\"" +
-      " ON \"res1\".\"TC_DATABASE\" = \"res2\".\"TC_DATABASE\"" +
-      " AND \"res1\".\"TC_TABLE\" = \"res2\".\"TC_TABLE\"" +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res2\".\"TC_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res2\".\"TC_PARTITION\" IS NULL)) " +
-      " LEFT JOIN " +
-          // Third sub-query - Gets the retry entries for corresponding db, table, partition.
-          "( SELECT \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", MAX(\"CQ_ID\") AS \"ID\", " +
-          " MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RETENTION\", " +
-          " MIN(\"CQ_COMMIT_TIME\") - %s + MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RECORD_CHECK\" FROM \"COMPACTION_QUEUE\" " +
-          " WHERE \"CQ_TYPE\" = " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"res3\" " +
-      " ON \"res1\".\"TC_DATABASE\" = \"res3\".\"CQ_DATABASE\" " +
-      " AND \"res1\".\"TC_TABLE\" = \"res3\".\"CQ_TABLE\" " +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res3\".\"CQ_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res3\".\"CQ_PARTITION\" IS NULL))" +
-      " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL";
-
-  private static final String DELETE_CQ_ENTRIES = "DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-
-  private static final String DELETE_CQ_AND_ABORT_ENTRIES = DELETE_CQ_ENTRIES +
-      " OR ( \"CQ_DATABASE\" = ? AND \"CQ_TABLE\" = ? " +
-      " AND \"CQ_PARTITION\" %s AND \"CQ_TYPE\" = " +
-      quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) + " )";
-
-  private static final String INSERT_ABORT_RETRY_ENTRY_INTO_COMPACTION_QUEUE =
-      "INSERT INTO \"COMPACTION_QUEUE\" (\"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
-      " \"CQ_TYPE\", \"CQ_STATE\", \"CQ_RETRY_RETENTION\", \"CQ_ERROR_MESSAGE\", \"CQ_COMMIT_TIME\") " +
-      " VALUES (?, ?, ?, ?, ?, ?, ?, ?, %s)";
+  
+  private static final Logger LOG = LoggerFactory.getLogger(CompactionTxnHandler.class.getName());
 
   public CompactionTxnHandler() {
   }
 
-  @Override
-  public void setConf(Configuration conf) {
-    super.setConf(conf);
-    synchronized (CompactionTxnHandler.class) {
-      if (connPoolCompaction == null) {
-        int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);

Review Comment:
   that part should stay here, see another comment with more details



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308768642


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -378,15 +393,21 @@ public void setConf(Configuration conf){
     int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
     synchronized (TxnHandler.class) {
       try (DataSourceProvider.DataSourceNameConfigurator configurator =
-               new DataSourceProvider.DataSourceNameConfigurator(conf, "txnhandler")) {
+               new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_TX)) {
         if (connPool == null) {
           connPool = setupJdbcConnectionPool(conf, maxPoolSize);
         }
         if (connPoolMutex == null) {
-          configurator.resetName("mutex");
+          configurator.resetName(POOL_MUTEX);
           connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize);
         }
       }
+      maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);
+      try (DataSourceProvider.DataSourceNameConfigurator ignored = new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_COMPACTOR)) {

Review Comment:
   are we initiating compaction pool even when CompactionTxnHandler is not used? CompactionTxnHandler is only used by compaction services, in other cases it's ThrowingTxnHandler.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308993832


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,38 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).execute(jdbcTemplate);

Review Comment:
   can we do 
   jdbcTemplate.execute(new MarkCleanedFunction(info, dbProduct, conf)) -> 
   function.execute(jdbcTemplate)



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310042003


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6246,18 +6303,24 @@ public void releaseLocks(Long timestamp) {
         LOG.warn("Unable to update MT_KEY2 value for MT_KEY1=" + key, ex);
         rollbackDBConn(dbConn);
       }
-      close(rs, stmt, dbConn);
+      TxnHandler.close(rs, stmt, dbConn);

Review Comment:
   why is  this change?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310118399


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6397,51 +6458,44 @@ private AbortCompactionResponseElement getAbortCompactionResponseElement(long co
 
   @RetrySemantics.SafeToRetry
   public AbortCompactionResponseElement abortCompaction(CompactionInfo compactionInfo) throws MetaException {
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolMutex);
-           PreparedStatement pStmt = dbConn.prepareStatement(TxnQueries.INSERT_INTO_COMPLETED_COMPACTION)) {
-        compactionInfo.state = TxnStore.ABORTED_STATE;
-        compactionInfo.errorMessage = "Comapction Aborted by Abort Comapction request.";
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, compactionInfo, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        if (updCount != 1) {
-          LOG.error("Unable to update compaction record: {}. updCnt={}", compactionInfo, updCount);
-          dbConn.rollback();
-          return getAbortCompactionResponseElement(compactionInfo.id, "Error",
-                  "Error while aborting compaction:Unable to update compaction record in COMPLETED_COMPACTIONS");
-        } else {
+    return retryHandler.executeWithRetry(new RetryCallProperties()
+            .withCallerId("abortCompaction(" + compactionInfo + ")")
+            .withDataSource(POOL_TX),
+        (DataSourceWrapper dataSourceWrapper) -> {
+          compactionInfo.state = TxnStore.ABORTED_STATE;
+          compactionInfo.errorMessage = "Comapction Aborted by Abort Comapction request.";
+          int updCount;
+          try {
+            updCount = dataSourceWrapper.execute(new InsertCompactionInfoCommand(compactionInfo, getDbTime().getTime()));
+          } catch (Exception e) {
+            LOG.error("Unable to update compaction record: {}.", compactionInfo);
+            return getAbortCompactionResponseElement(compactionInfo.id, "Error",
+                "Error while aborting compaction:Unable to update compaction record in COMPLETED_COMPACTIONS");
+          }
+          TransactionStatus status = dataSourceWrapper.getTransactionStatus();
           LOG.debug("Inserted {} entries into COMPLETED_COMPACTIONS", updCount);
-          try (PreparedStatement stmt = dbConn.prepareStatement("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?")) {
-            stmt.setLong(1, compactionInfo.id);
-            LOG.debug("Going to execute update on COMPACTION_QUEUE ");
-            updCount = stmt.executeUpdate();
+          try {
+            updCount = dataSourceWrapper.getJdbcTemplate().update("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id",
+                new MapSqlParameterSource().addValue("id", compactionInfo.id));
+
             if (updCount != 1) {
               LOG.error("Unable to update compaction record: {}. updCnt={}", compactionInfo, updCount);
-              dbConn.rollback();
+              status.setRollbackOnly();

Review Comment:
   Valid point, replaced with jdbcTemplate.rollback() to be consistent with others



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309954753


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1441,6 +1480,7 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
             // corresponding open txn event.
             LOG.info("Target txn id is missing for source txn id : {} and repl policy {}", sourceTxnId,
                 rqst.getReplPolicy());
+            jdbcTemplate.rollback();

Review Comment:
   Yes, jdbctemplate Txn requires explicit commit or rollback, because this is also a signal that the underlying connection can be released and put back to the pool. When using `@Transactional` annotation or the `TransactionTemplate` functional interface it is done by the framework. Methods annotated with @Retry also do the automatic commit/rollback, but since TxnHandler is not yet redesigned (I did only the must have modifications in it) here we need the direct approach for now.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310056717


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -3624,6 +3656,7 @@ public HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst
     }
   }
 
+  @Deprecated

Review Comment:
   why do we need to mark as deprecated and not just remove?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309803486


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java:
##########
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * Supports building resilient APIs by automatically retrying failed function calls.
+ */
+public class RetryHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+
+
+  private DataSourceWrapper jdbcTemplate;
+  private long deadlockRetryInterval;
+  private long retryInterval;
+  private int retryLimit;
+  
+  private Configuration conf;
+  
+  public static String getMessage(Exception ex) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(ex.getMessage());
+    if (ex instanceof SQLException) {
+      SQLException sqlEx = (SQLException)ex; 
+      sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")");
+    }
+    return sb.toString();
+  }
+  
+  public void init(Configuration conf, DataSourceWrapper jdbcTemplate){
+    this.conf = conf;
+    this.jdbcTemplate = jdbcTemplate;
+
+    retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+  }
+
+  /**
+   * Establishes a {@link DataSourceWrapper.RetryContext} and executes the passed {@link TransactionalFunction}. 
+   * Automatically retries the execution in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, retryLimit, ALLOWED_REPEATED_DEADLOCKS);
+  }
+
+  /**
+   * Executes the passed {@link TransactionalFunction}, without retry in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithoutRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, 0, 0);
+  }
+
+  private <Result> Result executeWithRetryInternal(RetryCallProperties properties, TransactionalFunction<Result> function
+      , int retryCount, int deadlockCount) throws MetaException {
+    Objects.requireNonNull(function, "RetryFunction<Result> cannot be null!");
+    Objects.requireNonNull(properties, "RetryCallProperties cannot be null!");
+
+    if (jdbcTemplate.hasRetryContext() && properties.getRetryPropagation().canJoinContext()) {
+      /*
+        If there is a RetryState in the ThreadLocal and we are allowed to join it, we can skip establishing a nested 
+        retry-call (and transaction).
+      */
+      LOG.info("Already established retry-context detected, current retry-call will join it. The passed CallProperties " +
+          "instance will be ignored, using the original one.");
+      try {
+        return function.execute(jdbcTemplate);
+      } catch (SQLException e) {
+        throw new UncategorizedSQLException(null, null, e);
+      }
+    }
+
+    if (!properties.getRetryPropagation().canCreateContext()) {
+      throw new MetaException("The current RetryPropagation mode (" + properties.getRetryPropagation().name() +
+          ") allows only to join an existing retry-context, but there is no context to join!");
+    }
+
+    LOG.debug("Running retry function:" + properties);
+
+    if (org.apache.commons.lang3.StringUtils.isEmpty(properties.getCaller())) {
+      properties.withCallerId(function.toString());
+    }
+
+    try {
+      if (properties.isLockInternally()) {
+        lockInternal();
+      }
+      try {
+        DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED);

Review Comment:
   Can't we have @Transaction annotation similar to Retry on method level where we could define the propagation?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310013878


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java:
##########
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * Supports building resilient APIs by automatically retrying failed function calls.
+ */
+public class RetryHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+
+
+  private DataSourceWrapper jdbcTemplate;
+  private long deadlockRetryInterval;
+  private long retryInterval;
+  private int retryLimit;
+  
+  private Configuration conf;
+  
+  public static String getMessage(Exception ex) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(ex.getMessage());
+    if (ex instanceof SQLException) {
+      SQLException sqlEx = (SQLException)ex; 
+      sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")");
+    }
+    return sb.toString();
+  }
+  
+  public void init(Configuration conf, DataSourceWrapper jdbcTemplate){
+    this.conf = conf;
+    this.jdbcTemplate = jdbcTemplate;
+
+    retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+  }
+
+  /**
+   * Establishes a {@link DataSourceWrapper.RetryContext} and executes the passed {@link TransactionalFunction}. 
+   * Automatically retries the execution in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, retryLimit, ALLOWED_REPEATED_DEADLOCKS);
+  }
+
+  /**
+   * Executes the passed {@link TransactionalFunction}, without retry in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithoutRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, 0, 0);
+  }
+
+  private <Result> Result executeWithRetryInternal(RetryCallProperties properties, TransactionalFunction<Result> function
+      , int retryCount, int deadlockCount) throws MetaException {
+    Objects.requireNonNull(function, "RetryFunction<Result> cannot be null!");
+    Objects.requireNonNull(properties, "RetryCallProperties cannot be null!");
+
+    if (jdbcTemplate.hasRetryContext() && properties.getRetryPropagation().canJoinContext()) {
+      /*
+        If there is a RetryState in the ThreadLocal and we are allowed to join it, we can skip establishing a nested 
+        retry-call (and transaction).
+      */
+      LOG.info("Already established retry-context detected, current retry-call will join it. The passed CallProperties " +
+          "instance will be ignored, using the original one.");
+      try {
+        return function.execute(jdbcTemplate);
+      } catch (SQLException e) {
+        throw new UncategorizedSQLException(null, null, e);
+      }
+    }
+
+    if (!properties.getRetryPropagation().canCreateContext()) {
+      throw new MetaException("The current RetryPropagation mode (" + properties.getRetryPropagation().name() +
+          ") allows only to join an existing retry-context, but there is no context to join!");
+    }
+
+    LOG.debug("Running retry function:" + properties);
+
+    if (org.apache.commons.lang3.StringUtils.isEmpty(properties.getCaller())) {
+      properties.withCallerId(function.toString());
+    }
+
+    try {
+      if (properties.isLockInternally()) {
+        lockInternal();
+      }
+      try {
+        DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED);

Review Comment:
   Created the propagation in RetryCallProperties. The idea was to tie together the retry and transaction handling: In case of failure rollback AND retry. In case of success commit AND do no retry. Handling them independently would allow more fine grained control, but would also introduce extra complexity: cases like doing a retry with the old connection and txn, or committing the txn, and still doing a retry. I'm not sure it's worth allowing these, as currently I cannot see places where such cases are necessary.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java:
##########
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * Supports building resilient APIs by automatically retrying failed function calls.
+ */
+public class RetryHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+
+
+  private DataSourceWrapper jdbcTemplate;
+  private long deadlockRetryInterval;
+  private long retryInterval;
+  private int retryLimit;
+  
+  private Configuration conf;
+  
+  public static String getMessage(Exception ex) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(ex.getMessage());
+    if (ex instanceof SQLException) {
+      SQLException sqlEx = (SQLException)ex; 
+      sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")");
+    }
+    return sb.toString();
+  }
+  
+  public void init(Configuration conf, DataSourceWrapper jdbcTemplate){
+    this.conf = conf;
+    this.jdbcTemplate = jdbcTemplate;
+
+    retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+  }
+
+  /**
+   * Establishes a {@link DataSourceWrapper.RetryContext} and executes the passed {@link TransactionalFunction}. 
+   * Automatically retries the execution in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, retryLimit, ALLOWED_REPEATED_DEADLOCKS);
+  }
+
+  /**
+   * Executes the passed {@link TransactionalFunction}, without retry in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithoutRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, 0, 0);
+  }
+
+  private <Result> Result executeWithRetryInternal(RetryCallProperties properties, TransactionalFunction<Result> function
+      , int retryCount, int deadlockCount) throws MetaException {
+    Objects.requireNonNull(function, "RetryFunction<Result> cannot be null!");
+    Objects.requireNonNull(properties, "RetryCallProperties cannot be null!");
+
+    if (jdbcTemplate.hasRetryContext() && properties.getRetryPropagation().canJoinContext()) {
+      /*
+        If there is a RetryState in the ThreadLocal and we are allowed to join it, we can skip establishing a nested 
+        retry-call (and transaction).
+      */
+      LOG.info("Already established retry-context detected, current retry-call will join it. The passed CallProperties " +
+          "instance will be ignored, using the original one.");
+      try {
+        return function.execute(jdbcTemplate);
+      } catch (SQLException e) {
+        throw new UncategorizedSQLException(null, null, e);
+      }
+    }
+
+    if (!properties.getRetryPropagation().canCreateContext()) {
+      throw new MetaException("The current RetryPropagation mode (" + properties.getRetryPropagation().name() +
+          ") allows only to join an existing retry-context, but there is no context to join!");
+    }
+
+    LOG.debug("Running retry function:" + properties);
+
+    if (org.apache.commons.lang3.StringUtils.isEmpty(properties.getCaller())) {
+      properties.withCallerId(function.toString());
+    }
+
+    try {
+      if (properties.isLockInternally()) {
+        lockInternal();
+      }
+      try {
+        DefaultTransactionDefinition transactionDefinition = new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED);

Review Comment:
   Created the propagation in `RetryCallProperties`. The idea was to tie together the retry and transaction handling: In case of failure rollback AND retry. In case of success commit AND do no retry. Handling them independently would allow more fine grained control, but would also introduce extra complexity: cases like doing a retry with the old connection and txn, or committing the txn, and still doing a retry. I'm not sure it's worth allowing these, as currently I cannot see places where such cases are necessary.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1226911088


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CallProperties.java:
##########
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.datanucleus.transaction.TransactionIsolation;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.sql.Connection;
+import java.util.function.Function;
+
+public class CallProperties {

Review Comment:
   This class holds the attributes of the retry call: error handler, transaction isolation level, rollback-on-error, etc. Using this class  there is no need to pass all the values as a separate parameter. Also, by setting the default values to the most common one, typically only 1-2 properties have to be set for a particular retry-call.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1610220721

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [67 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1615352791

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [79 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1616852689

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [80 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1678423758

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [67 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337244926


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5755,79 +5785,80 @@ private void timeOutLocks(Connection dbConn) {
    */
   @RetrySemantics.Idempotent
   public void performTimeOuts() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+    jdbcResource.bindDataSource(POOL_TX);
+    try (TransactionContext context = jdbcResource.getTransactionManager().getTransaction(PROPAGATION_REQUIRED)) {
       //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
       //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
       //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
       //would be to make sure someone cannot heartbeat one of these txns at the same time.
       //The attempt to heartbeat would block and fail immediately after it's unblocked.
       //With current (RC + multiple txns) implementation it is possible for someone to send
-      //heartbeat at the very end of the expire interval, and just after the Select from TXNS
+      //heartbeat at the very end of the expiry interval, and just after the Select from TXNS
       //is made, in which case heartbeat will succeed but txn will still be Aborted.
       //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
       //timely way.
-      timeOutLocks(dbConn);
-      while(true) {
-        stmt = dbConn.createStatement();
+      timeOutLocks();
+      while (true) {
         String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " AND (" +
-                "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
-             " OR " +
-                " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
-             ")";
+            " AND (" +
+            "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
+            " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
+            " OR " +
+            " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
+            " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
+            ")";
         //safety valve for extreme cases
         s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
+
         LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if(!rs.next()) {
-          return;//no more timedout txns
-        }
-        List<List<Long>> timedOutTxns = new ArrayList<>();
-        List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-        timedOutTxns.add(currentBatch);
-        do {
-          if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
-            currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-            timedOutTxns.add(currentBatch);
+        List<List<Long>> timedOutTxns = jdbcResource.getJdbcTemplate().query(s, rs -> {
+          List<List<Long>> txnbatch = new ArrayList<>();
+          List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+          while (rs.next()) {
+            currentBatch.add(rs.getLong(1));
+            if (currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
+              txnbatch.add(currentBatch);
+              currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+            }
           }
-          currentBatch.add(rs.getLong(1));
-        } while(rs.next());
-        dbConn.commit();
-        close(rs, stmt, null);
+          if (currentBatch.size() > 0) {
+            txnbatch.add(currentBatch);
+          }
+          return txnbatch;
+        });
+        //noinspection DataFlowIssue
+        if (timedOutTxns.size() == 0) {
+          jdbcResource.getTransactionManager().commit(context);

Review Comment:
   why is that needed? commit when nothing changed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337250536


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/AutoCloseableTransactionManager.java:
##########
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.jdbc;
+
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+/**
+ * A special TransactionManager which wraps a {@link PlatformTransactionManager} instance, to be able to wrap the
+ * returned {@link TransactionStatus} instances into {@link TransactionContext}. {@link TransactionContext} implements
+ * {@link AutoCloseable}, so it can be used in try-with-resources blocks. For more information see {@link TransactionContext}. 
+ * @see TransactionContext 
+ */
+public class AutoCloseableTransactionManager {

Review Comment:
   You are right, the name is misleading. Renamed to TransactionContextManager. The idea was to indicate somehow that this class returns AutoCloseable instances.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337241243


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java:
##########
@@ -461,9 +480,13 @@ void onRename(String oldCatName, String oldDbName, String oldTabName, String old
    * @return list of CompactionInfo structs.  These will not have id, type,
    * or runAs set since these are only potential compactions not actual ones.
    */
+  @SqlRetry
+  @Transactional(POOL_COMPACTOR)

Review Comment:
   The Method argument in ProxyTxnHandler points to the TxnStore method, there is no way to access the annotation if put on the implementing class.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293081957


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,39 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).call(dataSourceWrapper);
   }
-
+  
   /**
    * Clean up entries from TXN_TO_WRITE_ID table less than min_uncommited_txnid as found by
    * min(max(TXNS.txn_id), min(WRITE_SET.WS_COMMIT_ID), min(Aborted TXNS.txn_id)).
    */
   @Override
   @RetrySemantics.SafeToRetry
   public void cleanTxnToWriteIdTable() throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-
-      try {
-        long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
-
-        // We query for minimum values in all the queries and they can only increase by any concurrent
-        // operations. So, READ COMMITTED is sufficient.
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
-        // If there are no txns which are currently open or aborted in the system, then current value of
-        // max(TXNS.txn_id) could be min_uncommitted_txnid.
-        String s = "SELECT MIN(\"RES\".\"ID\") AS \"ID\" FROM (" +
-          " SELECT MAX(\"TXN_ID\") + 1 AS \"ID\" FROM \"TXNS\"" +
-          (useMinHistoryLevel ? "" :
-          "   UNION" +
-          " SELECT MIN(\"WS_TXNID\") AS \"ID\" FROM \"WRITE_SET\"") +
-          "   UNION" +
-          " SELECT MIN(\"TXN_ID\") AS \"ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          (useMinHistoryLevel ? "" :
-          "   OR \"TXN_STATE\" = " + TxnStatus.OPEN) +
-          " ) \"RES\"";
+    long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
 
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
-        }
-        long minUncommitedTxnid = Math.min(rs.getLong(1), minTxnIdSeenOpen);
-
-        // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
-        // to cleanup the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
-        s = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < " + minUncommitedTxnid;
-        LOG.debug("Going to execute delete <{}>", s);
-        int rc = stmt.executeUpdate(s);
-        LOG.info("Removed {} rows from TXN_TO_WRITE_ID with Txn Low-Water-Mark: {}", rc, minUncommitedTxnid);
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from TXN_TO_WRITE_ID table " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "cleanTxnToWriteIdTable");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      cleanTxnToWriteIdTable();
+    // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
+    // If there are no txns which are currently open or aborted in the system, then current value of
+    // max(TXNS.txn_id) could be min_uncommitted_txnid.
+    Long minTxnId = new MinUncommittedTxnIdHandler(useMinHistoryLevel).execute(dataSourceWrapper);
+    if (minTxnId == null) {
+      throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
     }
+    long minUncommitedTxnid = Math.min(minTxnId, minTxnIdSeenOpen);
+
+    // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
+    // to clean up the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
+    String sql = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < :txnId";
+    LOG.debug("Going to execute delete <{}>", sql);
+    int rc = dataSourceWrapper.getJdbcTemplate().update(sql, new MapSqlParameterSource("txnId", minUncommitedTxnid));

Review Comment:
   can't we use ParameterizedCommand here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293085665


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,38 +261,22 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction)) {
-        try (PreparedStatement stmt = dbConn.prepareStatement(DELETE_FAILED_TXNS_SQL)) {
-          //Aborted and committed are terminal states, so nothing about the txn can change
-          //after that, so READ COMMITTED is sufficient.
-          /*
-           * Only delete aborted / committed transaction in a way that guarantees two things:
-           * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
-           * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
-           */
-          long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId(dbConn);
-          stmt.setLong(1, lowWaterMark);
-          LOG.debug("Going to execute query <{}>", DELETE_FAILED_TXNS_SQL);
-          int rc = stmt.executeUpdate();
-          LOG.debug("Removed {} empty Aborted and Committed transactions from TXNS", rc);
-          LOG.debug("Going to commit");
-          dbConn.commit();
-        } catch (SQLException e) {
-          LOG.error("Unable to delete from txns table " + e.getMessage());
-          LOG.debug("Going to rollback");
-          rollbackDBConn(dbConn);
-          checkRetryable(e, "cleanEmptyAbortedTxns");
-          throw new MetaException("Unable to delete from txns table " + e.getMessage());
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e, "cleanEmptyAbortedTxns");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedAndCommittedTxns();
-    }
+    //Aborted and committed are terminal states, so nothing about the txn can change
+    //after that, so READ COMMITTED is sufficient.
+    /*
+     * Only delete aborted / committed transaction in a way that guarantees two things:
+     * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
+     * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
+     */
+    long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+    String query = "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +

Review Comment:
   why extract query from constants - DELETE_FAILED_TXNS_SQL?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337005969


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/FindPotentialCompactionsFunction.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.TransactionalFunction;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class FindPotentialCompactionsFunction implements TransactionalFunction<Set<CompactionInfo>> {
+  
+  private final int fetchSize;
+  private final int abortedThreshold;
+  private final long abortedTimeThreshold;
+  private final long lastChecked;
+  private final boolean collectAbortedTxns;
+
+  public FindPotentialCompactionsFunction(int fetchSize, int abortedThreshold, long abortedTimeThreshold, long lastChecked, boolean collectAbortedTxns) {
+    this.fetchSize = fetchSize;

Review Comment:
   this is a function, wouldn't it be better if we structure it as a Builder
   ````
   new FindPotentialCompactionsFunction()
     .withLastChecked(lastChecked)
     ......
     .withFetchSize(fetchSize)
     .execute(jdbcResource);
   ````



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337212502


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/jdbc/AutoCloseableTransactionManager.java:
##########
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.jdbc;
+
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+/**
+ * A special TransactionManager which wraps a {@link PlatformTransactionManager} instance, to be able to wrap the
+ * returned {@link TransactionStatus} instances into {@link TransactionContext}. {@link TransactionContext} implements
+ * {@link AutoCloseable}, so it can be used in try-with-resources blocks. For more information see {@link TransactionContext}. 
+ * @see TransactionContext 
+ */
+public class AutoCloseableTransactionManager {

Review Comment:
   why it's AutoCloseable if it doesn't implement Closeable



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 merged pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 merged PR #4384:
URL: https://github.com/apache/hive/pull/4384


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1226911088


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CallProperties.java:
##########
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.datanucleus.transaction.TransactionIsolation;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.sql.Connection;
+import java.util.function.Function;
+
+public class CallProperties {

Review Comment:
   This class holds the attributes/modifiers of the retry call: error handler, transaction isolation level, rollback-on-error, etc. Using this class  there is no need to pass all the values as a separate parameter. Also, by setting the default values to the most common one, typically only 1-2 properties have to be set for a particular retry-call.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293049419


##########
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java:
##########
@@ -742,12 +740,18 @@ public void testRevokeTimedOutWorkers() throws Exception {
   public void testFindPotentialCompactions() throws Exception {
     // Test that committing unlocks
     long txnid = openTxn();
-
-    List<LockComponent> components = new ArrayList<>();
-
-    components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "mytable", null, DataOperationType.UPDATE));
-    components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "yourtable", "mypartition=myvalue", DataOperationType.UPDATE));
-
+    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB,

Review Comment:
   why is this change? how is this related to the JIRA?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293054137


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java:
##########
@@ -79,8 +79,8 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
   public Set<Long> writeIds;
   public boolean hasUncompactedAborts;
 
-  byte[] metaInfo;
-  String hadoopJobId;
+  public byte[] metaInfo;
+  public String hadoopJobId;

Review Comment:
   looks like it's not in toString()



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1711328676

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [69 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] InvisibleProgrammer commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "InvisibleProgrammer (via GitHub)" <gi...@apache.org>.
InvisibleProgrammer commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1230711095


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/SqlParamSourceMapper.java:
##########
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+public interface SqlParamSourceMapper<T> {

Review Comment:
   Ok, thx



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308810407


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +93,12 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
+    Set<CompactionInfo> candidates = new HashSet<>(jdbcTemplate.execute(
+        new CompactionCandidateHandler(lastChecked, MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE))));
+    if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {

Review Comment:
   since we introduced wrappers, can't we move complete logic there (handle aborts)?
   btw COMPACTOR_FETCH_SIZE is not applied to aborts!!!



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308810407


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +93,12 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
+    Set<CompactionInfo> candidates = new HashSet<>(jdbcTemplate.execute(
+        new CompactionCandidateHandler(lastChecked, MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE))));
+    if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {

Review Comment:
   since we introduced wrappers, can't we move complete logic there (handle aborts)?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309775526


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;

Review Comment:
   shouldn't this be` threadLocal.get().transactionStatus`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309942542


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/RetryingTxnHandler.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryCallProperties;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.DataSourceWrapper;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.Retry;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Arrays;
+
+/**
+ * Responsible to use the {@link RetryHandler} to automatically retry the failed {@link TxnStore} methods if annotated with
+ * the {@link Retry} annotation.
+ */
+public class RetryingTxnHandler implements InvocationHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryingTxnHandler.class);
+
+  /**
+   * Gets the proxy interface for the given {@link TxnStore}.
+   * @param realStore The real {@link TxnStore} to proxy.
+   * @param retryHandler Responsible to re-execute the methods in case of failure.
+   * @return Returns the proxy object capable of retrying the failed calls automatically and transparently. 
+   */
+  public static TxnStore getProxy(TxnStore realStore, RetryHandler retryHandler) {
+    RetryingTxnHandler handler = new RetryingTxnHandler(realStore, retryHandler);
+    return (TxnStore) Proxy.newProxyInstance(
+        RetryingTxnHandler.class.getClassLoader(),
+        new Class[]{ TxnStore.class },
+        handler);
+  }
+
+  private final RetryHandler retryHandler;
+  private final TxnStore realStore;
+
+  private RetryingTxnHandler(TxnStore realStore, RetryHandler retryHandler) {
+    this.realStore = realStore;
+    this.retryHandler = retryHandler;
+  }
+
+  @Override
+  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    String strArgs = args == null ? "" : StringUtils.join(Arrays.asList(args), ",");
+    String callerId = method.getName() + "(" + strArgs + ")";
+    Retry retry = method.getAnnotation(Retry.class);
+    if (retry != null) {
+      RetryCallProperties properties = new RetryCallProperties()
+          .withCallerId(callerId)
+          .withDataSource(retry.datasource())
+          .withLockInternally(retry.lockInternally())
+          .withRollbackOnError(retry.rollbackOnError())
+          .withRetryOnDuplicateKey(retry.retryOnDuplicateKey())
+          .withTransactionIsolationLevel(retry.transactionIsolation());
+      return retryHandler.executeWithRetry(properties,
+          (DataSourceWrapper dataSourceWrapper) -> {
+            try {
+              LOG.info("Invoking method within retry context: {}", callerId);
+              return method.invoke(realStore, args);
+            } catch (IllegalAccessException | InvocationTargetException | UndeclaredThrowableException e) {
+              if (e.getCause() instanceof MetaException) {
+                throw (MetaException) e.getCause();

Review Comment:
   In the retry case we are inside the TransactionalFunction functional interface method which has a signature of `Result execute(DataSourceWrapper dataSourceWrapper) throws SQLException, MetaException;` Since e.getCause() returns Throwable it cannot be re-thrown directly. As a result we need 3 branches:
   - the cause was MetaException which is checked so cannot be handled by the runtime branch, we need to cast it
   - The cause was RuntimeException, we need to cast it 
   - in every other case we wrap it into a RuntimeException (not rethrowing the original).



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309954753


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1441,6 +1480,7 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
             // corresponding open txn event.
             LOG.info("Target txn id is missing for source txn id : {} and repl policy {}", sourceTxnId,
                 rqst.getReplPolicy());
+            jdbcTemplate.rollback();

Review Comment:
   Yes, jdbctemplate Txn requires explicit commit or rollback, because this is also a signal that the underlying connection can be released and put back to the pool. When using `@Transactional` annotation or the `TransactionTemplate` functional interface it is done by the framework, but I was using the lower lever direct approach to have fine grained control over txn and conneciton handling.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309791535


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;
+  }
+
+  /**
+   * @return Returns true if the current {@link Thread} has an associated {@link RetryContext}, false otherwise.
+   */
+  boolean hasRetryContext() {
+    return threadLocal.get() != null;
+  }
+
+  /**
+   * @return Returns the {@link NamedParameterJdbcTemplate} associated with the current {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public NamedParameterJdbcTemplate getJdbcTemplate() {
+    return jdbcTemplates.get(RetryContext.getRetryContext().datasource);
+  }
+
+  /**
+   * Gets a connection to the {@link DataSource} associated with the current {@link RetryContext}. Ensures that the same 
+   * instance is returned all the time within a particular transaction. Can be called only within an established 
+   * {@link RetryContext}. 
+   * @return Returns a connection to the datasource.
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public Connection getConnection() {
+    return DataSourceUtils.getConnection(dataSources.get(RetryContext.getRetryContext().datasource));
+  }
+
+  /**
+   * @return Returns a {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DatabaseProduct getDatabaseProduct() {
+    return databaseProduct;
+  }
+
+  /**
+   * <p>Establishes a {@link RetryContext} on the current {@link Thread}: Creates a new transaction with the given 
+   * {@link TransactionDefinition} using an internal {@link PlatformTransactionManager} associated with the reffered 
+   * {@link DataSource}. Returns a {@link TransactionStatus} representing the created transaction. The returned 
+   * {@link TransactionStatus} instance can be used for manual transaction handling.</p> 
+   * <br>
+   * <p>!!!! Please note that if there already is a {@link RetryContext} established, this call will create a nested</p>
+   * {@link RetryContext} (and transaction).</b>  
+   * @param definition The {@link TransactionDefinition} to use for creating a new transaction.
+   * @param dataSource The identifier of the {@link DataSource} to use
+   * @return Returns a {@link TransactionStatus} object representing the created transaction.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#getTransaction(TransactionDefinition)}
+   */
+  public TransactionStatus getTransactionWithinRetryContext(TransactionDefinition definition, String dataSource) throws TransactionException {
+    TransactionStatus status = transactionManagers.get(dataSource).getTransaction(definition);
+    RetryContext.setRetryContext(status, dataSource);
+    return status;
+  }
+
+  /**
+   * Commits the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#commit(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void commit() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();
+      transactionManagers.get(context.datasource).commit(context.transactionStatus);

Review Comment:
   there is a helper method `getTransactionStatus`



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1699911851

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [70 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309933181


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/RetryingTxnHandler.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryCallProperties;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.DataSourceWrapper;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.Retry;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Arrays;
+
+/**
+ * Responsible to use the {@link RetryHandler} to automatically retry the failed {@link TxnStore} methods if annotated with
+ * the {@link Retry} annotation.
+ */
+public class RetryingTxnHandler implements InvocationHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryingTxnHandler.class);
+
+  /**
+   * Gets the proxy interface for the given {@link TxnStore}.
+   * @param realStore The real {@link TxnStore} to proxy.
+   * @param retryHandler Responsible to re-execute the methods in case of failure.
+   * @return Returns the proxy object capable of retrying the failed calls automatically and transparently. 
+   */
+  public static TxnStore getProxy(TxnStore realStore, RetryHandler retryHandler) {
+    RetryingTxnHandler handler = new RetryingTxnHandler(realStore, retryHandler);
+    return (TxnStore) Proxy.newProxyInstance(
+        RetryingTxnHandler.class.getClassLoader(),
+        new Class[]{ TxnStore.class },
+        handler);
+  }
+
+  private final RetryHandler retryHandler;
+  private final TxnStore realStore;
+
+  private RetryingTxnHandler(TxnStore realStore, RetryHandler retryHandler) {
+    this.realStore = realStore;
+    this.retryHandler = retryHandler;
+  }
+
+  @Override
+  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    String strArgs = args == null ? "" : StringUtils.join(Arrays.asList(args), ",");
+    String callerId = method.getName() + "(" + strArgs + ")";
+    Retry retry = method.getAnnotation(Retry.class);
+    if (retry != null) {
+      RetryCallProperties properties = new RetryCallProperties()
+          .withCallerId(callerId)
+          .withDataSource(retry.datasource())
+          .withLockInternally(retry.lockInternally())
+          .withRollbackOnError(retry.rollbackOnError())
+          .withRetryOnDuplicateKey(retry.retryOnDuplicateKey())
+          .withTransactionIsolationLevel(retry.transactionIsolation());
+      return retryHandler.executeWithRetry(properties,
+          (DataSourceWrapper dataSourceWrapper) -> {
+            try {
+              LOG.info("Invoking method within retry context: {}", callerId);

Review Comment:
   Increased to debug



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309117627


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5695,24 +5759,18 @@ private List<LockInfo> getLocksFromLockId(Connection dbConn, long extLockId) thr
   // for read-only autoCommit=true statements.  This does a commit,
   // and thus should be done before any calls to heartbeat that will leave
   // open transactions.
-  private void timeOutLocks(Connection dbConn) {
-    Set<Long> timedOutLockIds = new TreeSet<>();
+  private void timeOutLocks() {
     //doing a SELECT first is less efficient but makes it easier to debug things
     //when txnid is <> 0, the lock is associated with a txn and is handled by performTimeOuts()
     //want to avoid expiring locks for a txn w/o expiring the txn itself
-    try (PreparedStatement pstmt = dbConn.prepareStatement(
-            String.format(SELECT_TIMED_OUT_LOCKS_QUERY, getEpochFn(dbProduct)))) {
-      pstmt.setLong(1, timeout);
-      LOG.debug("Going to execute query: <{}>", SELECT_TIMED_OUT_LOCKS_QUERY);
-      try (ResultSet rs = pstmt.executeQuery()) {
-        while (rs.next()) {
-          timedOutLockIds.add(rs.getLong(1));
-        }
-        dbConn.commit();
-        if (timedOutLockIds.isEmpty()) {
-          LOG.debug("Did not find any timed-out locks, therefore retuning.");
-          return;
-        }
+    try {
+      Set<Long> timedOutLockIds = new TreeSet<>(
+          jdbcTemplate.getJdbcTemplate().query(String.format(SELECT_TIMED_OUT_LOCKS_QUERY, getEpochFn(dbProduct)),

Review Comment:
   same question on txn propagation
   would we open a new connection here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309120157


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5755,80 +5807,86 @@ private void timeOutLocks(Connection dbConn) {
    */
   @RetrySemantics.Idempotent
   public void performTimeOuts() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
     try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-      //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
-      //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
-      //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
-      //would be to make sure someone cannot heartbeat one of these txns at the same time.
-      //The attempt to heartbeat would block and fail immediately after it's unblocked.
-      //With current (RC + multiple txns) implementation it is possible for someone to send
-      //heartbeat at the very end of the expire interval, and just after the Select from TXNS
-      //is made, in which case heartbeat will succeed but txn will still be Aborted.
-      //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
-      //timely way.
-      timeOutLocks(dbConn);
-      while(true) {
-        stmt = dbConn.createStatement();
-        String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " AND (" +
-                "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
-             " OR " +
-                " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
-             ")";
-        //safety valve for extreme cases
-        s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if(!rs.next()) {
-          return;//no more timedout txns
-        }
-        List<List<Long>> timedOutTxns = new ArrayList<>();
-        List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-        timedOutTxns.add(currentBatch);
-        do {
-          if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
-            currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-            timedOutTxns.add(currentBatch);
-          }
-          currentBatch.add(rs.getLong(1));
-        } while(rs.next());
-        dbConn.commit();
-        close(rs, stmt, null);
-        int numTxnsAborted = 0;
-        for(List<Long> batchToAbort : timedOutTxns) {
-          if (abortTxns(dbConn, batchToAbort, true, false, false, TxnErrorMsg.ABORT_TIMEOUT) == batchToAbort.size()) {
-            dbConn.commit();
-            numTxnsAborted += batchToAbort.size();
-            //todo: add TXNS.COMMENT filed and set it to 'aborted by system due to timeout'
-          }
-          else {
-            //could not abort all txns in this batch - this may happen because in parallel with this
-            //operation there was activity on one of the txns in this batch (commit/abort/heartbeat)
-            //This is not likely but may happen if client experiences long pause between heartbeats or
-            //unusually long/extreme pauses between heartbeat() calls and other logic in checkLock(),
-            //lock(), etc.
-            dbConn.rollback();
+      retryHandler.executeWithoutRetry(
+          new RetryCallProperties()
+              .withCallerId("performTimeOuts()")
+              .withDataSource(POOL_TX)
+              .withExceptionSupplier(e -> new MetaException("Aborting timed out transactions failed due to " + RetryHandler.getMessage(e))),
+          (DataSourceWrapper dataSourceWrapper) -> {

Review Comment:
   could wee extract the logic functioin into a lambda var and use it later in retryHandler?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309807148


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -378,15 +393,21 @@ public void setConf(Configuration conf){
     int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
     synchronized (TxnHandler.class) {
       try (DataSourceProvider.DataSourceNameConfigurator configurator =
-               new DataSourceProvider.DataSourceNameConfigurator(conf, "txnhandler")) {
+               new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_TX)) {
         if (connPool == null) {
           connPool = setupJdbcConnectionPool(conf, maxPoolSize);
         }
         if (connPoolMutex == null) {
-          configurator.resetName("mutex");
+          configurator.resetName(POOL_MUTEX);
           connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize);
         }
       }
+      maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);
+      try (DataSourceProvider.DataSourceNameConfigurator ignored = new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_COMPACTOR)) {

Review Comment:
   we'll be initiating compaction pool on every HMS instance, however, it should be only done on a leader 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1608035321

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [67 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1612063553

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1612733241

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [66 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309093634


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1441,6 +1480,7 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
             // corresponding open txn event.
             LOG.info("Target txn id is missing for source txn id : {} and repl policy {}", sourceTxnId,
                 rqst.getReplPolicy());
+            jdbcTemplate.rollback();

Review Comment:
   do you need to call rollback explicitly even when nothing changed?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310035765


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6397,51 +6458,44 @@ private AbortCompactionResponseElement getAbortCompactionResponseElement(long co
 
   @RetrySemantics.SafeToRetry
   public AbortCompactionResponseElement abortCompaction(CompactionInfo compactionInfo) throws MetaException {
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolMutex);
-           PreparedStatement pStmt = dbConn.prepareStatement(TxnQueries.INSERT_INTO_COMPLETED_COMPACTION)) {
-        compactionInfo.state = TxnStore.ABORTED_STATE;
-        compactionInfo.errorMessage = "Comapction Aborted by Abort Comapction request.";
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, compactionInfo, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        if (updCount != 1) {
-          LOG.error("Unable to update compaction record: {}. updCnt={}", compactionInfo, updCount);
-          dbConn.rollback();
-          return getAbortCompactionResponseElement(compactionInfo.id, "Error",
-                  "Error while aborting compaction:Unable to update compaction record in COMPLETED_COMPACTIONS");
-        } else {
+    return retryHandler.executeWithRetry(new RetryCallProperties()
+            .withCallerId("abortCompaction(" + compactionInfo + ")")
+            .withDataSource(POOL_TX),
+        (DataSourceWrapper dataSourceWrapper) -> {
+          compactionInfo.state = TxnStore.ABORTED_STATE;
+          compactionInfo.errorMessage = "Comapction Aborted by Abort Comapction request.";
+          int updCount;
+          try {
+            updCount = dataSourceWrapper.execute(new InsertCompactionInfoCommand(compactionInfo, getDbTime().getTime()));
+          } catch (Exception e) {
+            LOG.error("Unable to update compaction record: {}.", compactionInfo);
+            return getAbortCompactionResponseElement(compactionInfo.id, "Error",
+                "Error while aborting compaction:Unable to update compaction record in COMPLETED_COMPACTIONS");
+          }
+          TransactionStatus status = dataSourceWrapper.getTransactionStatus();
           LOG.debug("Inserted {} entries into COMPLETED_COMPACTIONS", updCount);
-          try (PreparedStatement stmt = dbConn.prepareStatement("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?")) {
-            stmt.setLong(1, compactionInfo.id);
-            LOG.debug("Going to execute update on COMPACTION_QUEUE ");
-            updCount = stmt.executeUpdate();
+          try {
+            updCount = dataSourceWrapper.getJdbcTemplate().update("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id",
+                new MapSqlParameterSource().addValue("id", compactionInfo.id));
+
             if (updCount != 1) {
               LOG.error("Unable to update compaction record: {}. updCnt={}", compactionInfo, updCount);
-              dbConn.rollback();
+              status.setRollbackOnly();

Review Comment:
   with that approach it's hard to follow the db txn lifecycle. where do we commit or rollback the txn? inside of retryhandler



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310042003


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6246,18 +6303,24 @@ public void releaseLocks(Long timestamp) {
         LOG.warn("Unable to update MT_KEY2 value for MT_KEY1=" + key, ex);
         rollbackDBConn(dbConn);
       }
-      close(rs, stmt, dbConn);
+      TxnHandler.close(rs, stmt, dbConn);

Review Comment:
   why is  this change?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309030905


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/RetryingTxnHandler.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryCallProperties;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.DataSourceWrapper;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.Retry;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.RetryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Arrays;
+
+/**
+ * Responsible to use the {@link RetryHandler} to automatically retry the failed {@link TxnStore} methods if annotated with
+ * the {@link Retry} annotation.
+ */
+public class RetryingTxnHandler implements InvocationHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryingTxnHandler.class);
+
+  /**
+   * Gets the proxy interface for the given {@link TxnStore}.
+   * @param realStore The real {@link TxnStore} to proxy.
+   * @param retryHandler Responsible to re-execute the methods in case of failure.
+   * @return Returns the proxy object capable of retrying the failed calls automatically and transparently. 
+   */
+  public static TxnStore getProxy(TxnStore realStore, RetryHandler retryHandler) {
+    RetryingTxnHandler handler = new RetryingTxnHandler(realStore, retryHandler);
+    return (TxnStore) Proxy.newProxyInstance(
+        RetryingTxnHandler.class.getClassLoader(),
+        new Class[]{ TxnStore.class },
+        handler);
+  }
+
+  private final RetryHandler retryHandler;
+  private final TxnStore realStore;
+
+  private RetryingTxnHandler(TxnStore realStore, RetryHandler retryHandler) {
+    this.realStore = realStore;
+    this.retryHandler = retryHandler;
+  }
+
+  @Override
+  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    String strArgs = args == null ? "" : StringUtils.join(Arrays.asList(args), ",");
+    String callerId = method.getName() + "(" + strArgs + ")";
+    Retry retry = method.getAnnotation(Retry.class);
+    if (retry != null) {
+      RetryCallProperties properties = new RetryCallProperties()
+          .withCallerId(callerId)
+          .withDataSource(retry.datasource())
+          .withLockInternally(retry.lockInternally())
+          .withRollbackOnError(retry.rollbackOnError())
+          .withRetryOnDuplicateKey(retry.retryOnDuplicateKey())
+          .withTransactionIsolationLevel(retry.transactionIsolation());
+      return retryHandler.executeWithRetry(properties,
+          (DataSourceWrapper dataSourceWrapper) -> {
+            try {
+              LOG.info("Invoking method within retry context: {}", callerId);
+              return method.invoke(realStore, args);
+            } catch (IllegalAccessException | InvocationTargetException | UndeclaredThrowableException e) {
+              if (e.getCause() instanceof MetaException) {
+                throw (MetaException) e.getCause();

Review Comment:
   what is the purpose of casting here? same for Runtime exceptiion. check same error handling for non-retry



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310023851


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -3645,6 +3678,38 @@ long generateCompactionQueueId(Statement stmt) throws SQLException, MetaExceptio
     }
   }
 
+  long generateCompactionQueueId() throws MetaException {
+    // Get the id for the next entry in the queue
+    String sql = sqlGenerator.addForUpdateClause("SELECT \"NCQ_NEXT\" FROM \"NEXT_COMPACTION_QUEUE_ID\"");

Review Comment:
   No, it's executed in the same. The `TransactionStatus` and the `RetryContext` are stored in `ThreadLocal`, so they are accessible throughout the execution until a commit/rollback happens.
   `jdbcTemplate.getJdbcTemplate() `will access the same connection as the caller did/will.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5695,24 +5759,18 @@ private List<LockInfo> getLocksFromLockId(Connection dbConn, long extLockId) thr
   // for read-only autoCommit=true statements.  This does a commit,
   // and thus should be done before any calls to heartbeat that will leave
   // open transactions.
-  private void timeOutLocks(Connection dbConn) {
-    Set<Long> timedOutLockIds = new TreeSet<>();
+  private void timeOutLocks() {
     //doing a SELECT first is less efficient but makes it easier to debug things
     //when txnid is <> 0, the lock is associated with a txn and is handled by performTimeOuts()
     //want to avoid expiring locks for a txn w/o expiring the txn itself
-    try (PreparedStatement pstmt = dbConn.prepareStatement(
-            String.format(SELECT_TIMED_OUT_LOCKS_QUERY, getEpochFn(dbProduct)))) {
-      pstmt.setLong(1, timeout);
-      LOG.debug("Going to execute query: <{}>", SELECT_TIMED_OUT_LOCKS_QUERY);
-      try (ResultSet rs = pstmt.executeQuery()) {
-        while (rs.next()) {
-          timedOutLockIds.add(rs.getLong(1));
-        }
-        dbConn.commit();
-        if (timedOutLockIds.isEmpty()) {
-          LOG.debug("Did not find any timed-out locks, therefore retuning.");
-          return;
-        }
+    try {
+      Set<Long> timedOutLockIds = new TreeSet<>(
+          jdbcTemplate.getJdbcTemplate().query(String.format(SELECT_TIMED_OUT_LOCKS_QUERY, getEpochFn(dbProduct)),

Review Comment:
   same as above



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308768642


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -378,15 +393,21 @@ public void setConf(Configuration conf){
     int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
     synchronized (TxnHandler.class) {
       try (DataSourceProvider.DataSourceNameConfigurator configurator =
-               new DataSourceProvider.DataSourceNameConfigurator(conf, "txnhandler")) {
+               new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_TX)) {
         if (connPool == null) {
           connPool = setupJdbcConnectionPool(conf, maxPoolSize);
         }
         if (connPoolMutex == null) {
-          configurator.resetName("mutex");
+          configurator.resetName(POOL_MUTEX);
           connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize);
         }
       }
+      maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);
+      try (DataSourceProvider.DataSourceNameConfigurator ignored = new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_COMPACTOR)) {

Review Comment:
   are we initiating compaction pool even when CompactionTxnHandler is not used?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309117627


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5695,24 +5759,18 @@ private List<LockInfo> getLocksFromLockId(Connection dbConn, long extLockId) thr
   // for read-only autoCommit=true statements.  This does a commit,
   // and thus should be done before any calls to heartbeat that will leave
   // open transactions.
-  private void timeOutLocks(Connection dbConn) {
-    Set<Long> timedOutLockIds = new TreeSet<>();
+  private void timeOutLocks() {
     //doing a SELECT first is less efficient but makes it easier to debug things
     //when txnid is <> 0, the lock is associated with a txn and is handled by performTimeOuts()
     //want to avoid expiring locks for a txn w/o expiring the txn itself
-    try (PreparedStatement pstmt = dbConn.prepareStatement(
-            String.format(SELECT_TIMED_OUT_LOCKS_QUERY, getEpochFn(dbProduct)))) {
-      pstmt.setLong(1, timeout);
-      LOG.debug("Going to execute query: <{}>", SELECT_TIMED_OUT_LOCKS_QUERY);
-      try (ResultSet rs = pstmt.executeQuery()) {
-        while (rs.next()) {
-          timedOutLockIds.add(rs.getLong(1));
-        }
-        dbConn.commit();
-        if (timedOutLockIds.isEmpty()) {
-          LOG.debug("Did not find any timed-out locks, therefore retuning.");
-          return;
-        }
+    try {
+      Set<Long> timedOutLockIds = new TreeSet<>(
+          jdbcTemplate.getJdbcTemplate().query(String.format(SELECT_TIMED_OUT_LOCKS_QUERY, getEpochFn(dbProduct)),

Review Comment:
   same question on txn propagation



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309808549


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -473,6 +505,18 @@ private boolean checkIfTableIsUsable(String tableName, boolean configValue) thro
     }
     return tableExists;
   }
+  
+  @Override
+  @RetrySemantics.ReadOnly
+  public DatabaseProduct getDatabaseProduct() {

Review Comment:
   why do we need this? product is available in dataSourceWrapper



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309917798


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -289,109 +125,12 @@ public CompactionInfo findNextToCompact(String workerId) throws MetaException {
    */
   @Override
   @RetrySemantics.SafeToRetry
-  @SuppressWarnings("squid:S2095")
   public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws MetaException {
-    try {
-      if (rqst == null) {
-        throw new MetaException("FindNextCompactRequest is null");
-      }
-
-      Connection dbConn = null;
-      PreparedStatement stmt = null;
-      //need a separate stmt for executeUpdate() otherwise it will close the ResultSet(HIVE-12725)
-      Statement updStmt = null;
-      ResultSet rs = null;
-
-      long poolTimeout = MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_WORKER_POOL_TIMEOUT, TimeUnit.MILLISECONDS);
-
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        StringBuilder sb = new StringBuilder();
-        sb.append("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
-          "\"CQ_TYPE\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\", " +
-          "\"CQ_TBLPROPERTIES\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_STATE\" = " + quoteChar(INITIATED_STATE) + " AND ");
-        boolean hasPoolName = StringUtils.isNotBlank(rqst.getPoolName());
-        if(hasPoolName) {
-          sb.append("\"CQ_POOL_NAME\"=?");
-        } else {
-          sb.append("\"CQ_POOL_NAME\" IS NULL OR  \"CQ_ENQUEUE_TIME\" < (")
-            .append(getEpochFn(dbProduct)).append(" - ").append(poolTimeout).append(")");
-        }
-        sb.append(" ORDER BY \"CQ_ID\" ASC");
-        String query = sb.toString();
-        stmt = dbConn.prepareStatement(query);
-        if (hasPoolName) {
-          stmt.setString(1, rqst.getPoolName());
-        }
-
-        LOG.debug("Going to execute query <{}>", query);
-        rs = stmt.executeQuery();
-        if (!rs.next()) {
-          LOG.debug("No compactions found ready to compact");
-          dbConn.rollback();
-          return null;
-        }
-        updStmt = dbConn.createStatement();
-        do {
-          CompactionInfo info = new CompactionInfo();
-          info.id = rs.getLong(1);
-          info.dbname = rs.getString(2);
-          info.tableName = rs.getString(3);
-          info.partName = rs.getString(4);
-          info.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(5).charAt(0));
-          info.poolName = rs.getString(6);
-          info.numberOfBuckets = rs.getInt(7);
-          info.orderByClause = rs.getString(8);
-          info.properties = rs.getString(9);
-          info.workerId = rqst.getWorkerId();
-
-          String workerId = rqst.getWorkerId();
-          String workerVersion = rqst.getWorkerVersion();
-          String workerIdSqlValue = (workerId == null) ? "NULL" : ("'" + workerId + "'");
-
-          // Now, update this record as being worked on by this worker.
-          long now = getDbTime(dbConn);
-          query = "" +
-              "UPDATE " +
-              "  \"COMPACTION_QUEUE\" " +
-              "SET " +
-              " \"CQ_WORKER_ID\" = " + workerIdSqlValue + ", " +
-              " \"CQ_WORKER_VERSION\" = '" + workerVersion + "', " +
-              " \"CQ_START\" = " + now + ", " +
-              " \"CQ_STATE\" = '" + WORKING_STATE + "' " +
-              "WHERE \"CQ_ID\" = " + info.id +
-              "  AND \"CQ_STATE\"='" + INITIATED_STATE + "'";
-
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = updStmt.executeUpdate(query);
-          if(updCount == 1) {
-            dbConn.commit();
-            return info;
-          }
-          if(updCount == 0) {
-            LOG.debug("Worker {} (version: {}) picked up {}", workerId, workerVersion, info);
-            continue;
-          }
-          LOG.error("Unable to set to cq_state={} for compaction record: {}. updCnt={}. workerId={}. workerVersion={}",
-              WORKING_STATE, info, updCount, workerId, workerVersion);
-          dbConn.rollback();
-          return null;
-        } while( rs.next());
-        dbConn.rollback();
-        return null;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for compaction, " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "findNextToCompact(rqst:" + rqst + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(updStmt);
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return findNextToCompact(rqst);
+    if (rqst == null) {
+      throw new MetaException("FindNextCompactRequest is null");
     }
+    long poolTimeout = MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_WORKER_POOL_TIMEOUT, TimeUnit.MILLISECONDS);

Review Comment:
   This timeout is specific to this query.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310005204


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));

Review Comment:
   Moved to registerDataSource methods and fixed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1659803006

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [83 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1230721699


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state
+    try {

Review Comment:
   Yeah I already rewrote it to use column names. My answer was to @aturoczy regarding spring JDBC's automatic object mapping via  `BeanPropertyRowMapper<T>` or `DataClassRowMapper<T>`. I consider those fragile as there is an implicit mapping between column names and field names. Another solution would be some annotation or external mapping based ORM framework, but that would require separated DAO objects instead of using `CompactionInfo` everywhere. This PR will be huge anyways, so even if we wanto to go forward with one of those frameworks, I would do that separately.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1230707069


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/handler/AbortedTxnCleaner.java:
##########
@@ -99,7 +100,7 @@ private void clean(CompactionInfo info, long minOpenWriteTxn, boolean metricsEna
       if (isNull(t)) {
         // The table was dropped before we got around to cleaning it.
         LOG.info("Unable to find table {}, assuming it was dropped.", info.getFullTableName());
-        txnHandler.markCleaned(info, true);
+        txnHandler.markCleaned((CompactionInfo) info, true);

Review Comment:
   Removed the AbortedTxnInfo class, it will be introduced together with the thrift changes



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337247983


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -5755,79 +5785,80 @@ private void timeOutLocks(Connection dbConn) {
    */
   @RetrySemantics.Idempotent
   public void performTimeOuts() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+    jdbcResource.bindDataSource(POOL_TX);
+    try (TransactionContext context = jdbcResource.getTransactionManager().getTransaction(PROPAGATION_REQUIRED)) {
       //We currently commit after selecting the TXNS to abort.  So whether SERIALIZABLE
       //READ_COMMITTED, the effect is the same.  We could use FOR UPDATE on Select from TXNS
       //and do the whole performTimeOuts() in a single huge transaction, but the only benefit
       //would be to make sure someone cannot heartbeat one of these txns at the same time.
       //The attempt to heartbeat would block and fail immediately after it's unblocked.
       //With current (RC + multiple txns) implementation it is possible for someone to send
-      //heartbeat at the very end of the expire interval, and just after the Select from TXNS
+      //heartbeat at the very end of the expiry interval, and just after the Select from TXNS
       //is made, in which case heartbeat will succeed but txn will still be Aborted.
       //Solving this corner case is not worth the perf penalty.  The client should heartbeat in a
       //timely way.
-      timeOutLocks(dbConn);
-      while(true) {
-        stmt = dbConn.createStatement();
+      timeOutLocks();
+      while (true) {
         String s = " \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " AND (" +
-                "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
-             " OR " +
-                " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
-                " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
-             ")";
+            " AND (" +
+            "\"TXN_TYPE\" != " + TxnType.REPL_CREATED.getValue() +
+            " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + timeout +
+            " OR " +
+            " \"TXN_TYPE\" = " + TxnType.REPL_CREATED.getValue() +
+            " AND \"TXN_LAST_HEARTBEAT\" <  " + getEpochFn(dbProduct) + "-" + replicationTxnTimeout +
+            ")";
         //safety valve for extreme cases
         s = sqlGenerator.addLimitClause(10 * TIMED_OUT_TXN_ABORT_BATCH_SIZE, s);
+
         LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if(!rs.next()) {
-          return;//no more timedout txns
-        }
-        List<List<Long>> timedOutTxns = new ArrayList<>();
-        List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-        timedOutTxns.add(currentBatch);
-        do {
-          if(currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
-            currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
-            timedOutTxns.add(currentBatch);
+        List<List<Long>> timedOutTxns = jdbcResource.getJdbcTemplate().query(s, rs -> {
+          List<List<Long>> txnbatch = new ArrayList<>();
+          List<Long> currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+          while (rs.next()) {
+            currentBatch.add(rs.getLong(1));
+            if (currentBatch.size() == TIMED_OUT_TXN_ABORT_BATCH_SIZE) {
+              txnbatch.add(currentBatch);
+              currentBatch = new ArrayList<>(TIMED_OUT_TXN_ABORT_BATCH_SIZE);
+            }
           }
-          currentBatch.add(rs.getLong(1));
-        } while(rs.next());
-        dbConn.commit();
-        close(rs, stmt, null);
+          if (currentBatch.size() > 0) {
+            txnbatch.add(currentBatch);
+          }
+          return txnbatch;
+        });
+        //noinspection DataFlowIssue
+        if (timedOutTxns.size() == 0) {
+          jdbcResource.getTransactionManager().commit(context);

Review Comment:
   This is inside a loop, we get here when there is no more records to process, but there are processed ones to commit.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337011321


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/NextCompactionHandler.java:
##########
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.FindNextCompactRequest;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.QueryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+
+import static org.apache.hadoop.hive.metastore.txn.TxnStore.INITIATED_STATE;
+import static org.apache.hadoop.hive.metastore.txn.TxnStore.WORKING_STATE;
+import static org.apache.hadoop.hive.metastore.txn.TxnUtils.getEpochFn;
+
+public class NextCompactionHandler implements QueryHandler<CompactionInfo> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(NextCompactionHandler.class);
+
+  private final FindNextCompactRequest request;
+  private final MultiDataSourceJdbcResource retriableDatasourceTxnManager;
+  private final Timestamp currentDbTime;
+  private final long poolTimeout;
+
+  private static final String updateStatement =
+      "UPDATE \"COMPACTION_QUEUE\" " +
+          "SET " +
+          " \"CQ_WORKER_ID\" = :workerId, " +
+          " \"CQ_WORKER_VERSION\" = :workerVersion, " +
+          " \"CQ_START\" = :now, " +
+          " \"CQ_STATE\" = :newState " +
+          "WHERE \"CQ_ID\" = :id AND \"CQ_STATE\"= :oldState";
+
+  public NextCompactionHandler(FindNextCompactRequest request, MultiDataSourceJdbcResource retriableDatasourceTxnManager, Timestamp currentDbTime, long poolTimeout) {
+    this.request = request;
+    this.retriableDatasourceTxnManager = retriableDatasourceTxnManager;
+    this.currentDbTime = currentDbTime;
+    this.poolTimeout = poolTimeout;
+  }
+
+  @Override
+  public String getParameterizedQueryString(DatabaseProduct databaseProduct) throws MetaException {
+    StringBuilder sb = new StringBuilder();
+    sb.append("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
+        "\"CQ_TYPE\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\", " +
+        "\"CQ_TBLPROPERTIES\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_STATE\" = :state AND ");
+    boolean hasPoolName = StringUtils.isNotBlank(request.getPoolName());
+    if(hasPoolName) {
+      sb.append("\"CQ_POOL_NAME\"= :poolName");
+    } else {
+      sb.append("\"CQ_POOL_NAME\" IS NULL OR  \"CQ_ENQUEUE_TIME\" < (")
+          .append(getEpochFn(databaseProduct)).append(" - ").append(poolTimeout).append(")");
+    }
+    sb.append(" ORDER BY \"CQ_ID\" ASC");
+    return sb.toString();
+  }
+
+  @Override
+  public SqlParameterSource getQueryParameters() {
+    MapSqlParameterSource params = new MapSqlParameterSource()
+        .addValue("state", Character.toString(INITIATED_STATE), Types.CHAR);
+    if(StringUtils.isNotBlank(request.getPoolName())) {
+      params.addValue("poolName", request.getPoolName());
+    }
+    return params;
+  }
+
+  @Override
+  public CompactionInfo extractData(ResultSet rs) throws SQLException, DataAccessException {
+    CompactionInfo info;
+    while (rs.next()){

Review Comment:
   should be if



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1710410980

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [69 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1712113521

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [76 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293096460


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/TransactionalVoidFunction.java:
##########
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+import java.sql.SQLException;
+
+/**
+ * A functional interface representing a function call (typically a statement) which is done within a database transaction.
+ */
+@FunctionalInterface
+public interface TransactionalVoidFunction {
+
+  /**
+   * Implementations typically should execute transsactional database calls inside.
+   * @param dataSourceWrapper A {@link DataSourceWrapper} instance responsible for providing all the necessary resources 
+   *                          to be able to perform transactional database calls.
+   * @throws SQLException Thrown if any of the JDBC calls fail
+   * @throws MetaException Thrown in case of application error within the function
+   */
+  void call(DataSourceWrapper dataSourceWrapper) throws SQLException, MetaException;

Review Comment:
   why do you need a separate interface for that, you already have a generic TransactionalFunction<Void>



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293202181


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/TransactionalVoidFunction.java:
##########
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+import java.sql.SQLException;
+
+/**
+ * A functional interface representing a function call (typically a statement) which is done within a database transaction.
+ */
+@FunctionalInterface
+public interface TransactionalVoidFunction {
+
+  /**
+   * Implementations typically should execute transsactional database calls inside.
+   * @param dataSourceWrapper A {@link DataSourceWrapper} instance responsible for providing all the necessary resources 
+   *                          to be able to perform transactional database calls.
+   * @throws SQLException Thrown if any of the JDBC calls fail
+   * @throws MetaException Thrown in case of application error within the function
+   */
+  void call(DataSourceWrapper dataSourceWrapper) throws SQLException, MetaException;

Review Comment:
   removed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293108300


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,39 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).call(dataSourceWrapper);
   }
-
+  
   /**
    * Clean up entries from TXN_TO_WRITE_ID table less than min_uncommited_txnid as found by
    * min(max(TXNS.txn_id), min(WRITE_SET.WS_COMMIT_ID), min(Aborted TXNS.txn_id)).
    */
   @Override
   @RetrySemantics.SafeToRetry
   public void cleanTxnToWriteIdTable() throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-
-      try {
-        long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
-
-        // We query for minimum values in all the queries and they can only increase by any concurrent
-        // operations. So, READ COMMITTED is sufficient.
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
-        // If there are no txns which are currently open or aborted in the system, then current value of
-        // max(TXNS.txn_id) could be min_uncommitted_txnid.
-        String s = "SELECT MIN(\"RES\".\"ID\") AS \"ID\" FROM (" +
-          " SELECT MAX(\"TXN_ID\") + 1 AS \"ID\" FROM \"TXNS\"" +
-          (useMinHistoryLevel ? "" :
-          "   UNION" +
-          " SELECT MIN(\"WS_TXNID\") AS \"ID\" FROM \"WRITE_SET\"") +
-          "   UNION" +
-          " SELECT MIN(\"TXN_ID\") AS \"ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          (useMinHistoryLevel ? "" :
-          "   OR \"TXN_STATE\" = " + TxnStatus.OPEN) +
-          " ) \"RES\"";
+    long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
 
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
-        }
-        long minUncommitedTxnid = Math.min(rs.getLong(1), minTxnIdSeenOpen);
-
-        // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
-        // to cleanup the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
-        s = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < " + minUncommitedTxnid;
-        LOG.debug("Going to execute delete <{}>", s);
-        int rc = stmt.executeUpdate(s);
-        LOG.info("Removed {} rows from TXN_TO_WRITE_ID with Txn Low-Water-Mark: {}", rc, minUncommitedTxnid);
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from TXN_TO_WRITE_ID table " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "cleanTxnToWriteIdTable");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      cleanTxnToWriteIdTable();
+    // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
+    // If there are no txns which are currently open or aborted in the system, then current value of
+    // max(TXNS.txn_id) could be min_uncommitted_txnid.
+    Long minTxnId = new MinUncommittedTxnIdHandler(useMinHistoryLevel).execute(dataSourceWrapper);
+    if (minTxnId == null) {
+      throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
     }
+    long minUncommitedTxnid = Math.min(minTxnId, minTxnIdSeenOpen);
+
+    // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
+    // to clean up the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
+    String sql = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < :txnId";
+    LOG.debug("Going to execute delete <{}>", sql);
+    int rc = dataSourceWrapper.getJdbcTemplate().update(sql, new MapSqlParameterSource("txnId", minUncommitedTxnid));

Review Comment:
   Sure, we can. 



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,38 +261,22 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction)) {
-        try (PreparedStatement stmt = dbConn.prepareStatement(DELETE_FAILED_TXNS_SQL)) {
-          //Aborted and committed are terminal states, so nothing about the txn can change
-          //after that, so READ COMMITTED is sufficient.
-          /*
-           * Only delete aborted / committed transaction in a way that guarantees two things:
-           * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
-           * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
-           */
-          long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId(dbConn);
-          stmt.setLong(1, lowWaterMark);
-          LOG.debug("Going to execute query <{}>", DELETE_FAILED_TXNS_SQL);
-          int rc = stmt.executeUpdate();
-          LOG.debug("Removed {} empty Aborted and Committed transactions from TXNS", rc);
-          LOG.debug("Going to commit");
-          dbConn.commit();
-        } catch (SQLException e) {
-          LOG.error("Unable to delete from txns table " + e.getMessage());
-          LOG.debug("Going to rollback");
-          rollbackDBConn(dbConn);
-          checkRetryable(e, "cleanEmptyAbortedTxns");
-          throw new MetaException("Unable to delete from txns table " + e.getMessage());
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e, "cleanEmptyAbortedTxns");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedAndCommittedTxns();
-    }
+    //Aborted and committed are terminal states, so nothing about the txn can change
+    //after that, so READ COMMITTED is sufficient.
+    /*
+     * Only delete aborted / committed transaction in a way that guarantees two things:
+     * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
+     * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
+     */
+    long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+    String query = "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
+        "AND (\"TXN_STATE\" = :abortedState OR \"TXN_STATE\" = :committedState) AND \"TXN_ID\" < :txnId";
+
+    LOG.debug("Going to execute query <{}>", query);
+    dataSourceWrapper.getJdbcTemplate().update(query, new MapSqlParameterSource()

Review Comment:
   Sure, we can.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293134630


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1046,37 +290,14 @@ public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
    */
   @Override
   @RetrySemantics.Idempotent
-  public void revokeFromLocalWorkers(String hostname) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL, \"CQ_STATE\" = '"
-          + INITIATED_STATE+ "' WHERE \"CQ_STATE\" = '" + WORKING_STATE + "' AND \"CQ_WORKER_ID\" LIKE '"
-          +  hostname + "%'";
-        LOG.debug("Going to execute update <{}>", s);
-        // It isn't an error if the following returns no rows, as the local workers could have died
-        // with  nothing assigned to them.
-        int updated = stmt.executeUpdate(s);
-        LOG.debug("Set {} compaction queue entries to {} state for host {}", updated, INITIATED_RESPONSE, hostname);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to change dead worker's records back to initiated state " +
-          e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "revokeFromLocalWorkers(hostname:" + hostname +")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      revokeFromLocalWorkers(hostname);
-    }
+  public void revokeFromLocalWorkers(String hostname) {
+    dataSourceWrapper.getJdbcTemplate().update(
+        "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL," +

Review Comment:
   Answered above, if necessary I can resture the constant here as well



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1615750428

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [79 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1621682126

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [81 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1659404707

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [83 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1296883919


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java:
##########
@@ -461,9 +479,11 @@ void onRename(String oldCatName, String oldDbName, String oldTabName, String old
    * @return list of CompactionInfo structs.  These will not have id, type,
    * or runAs set since these are only potential compactions not actual ones.
    */
+  @Retry(datasource = POOL_COMPACTOR)
   @RetrySemantics.ReadOnly

Review Comment:
   As far as I understand yes, that is related to hs2 - hms communication. There is some retry mechanism there as well.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1623229630

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [84 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309772236


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {

Review Comment:
   nit: space



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309768306


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryCallProperties.java:
##########
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.datanucleus.transaction.TransactionIsolation;
+
+import java.util.function.Function;
+
+/**
+ * Contains all the properties which can alter the behavior of a retry-call in 
+ * {@link org.apache.hadoop.hive.metastore.txn.retryhandling.RetryHandler}.
+ */
+public class RetryCallProperties {
+
+  private String dataSource;
+  private String caller = null;
+  private boolean retryOnDuplicateKey = false;
+  private int transactionIsolationLevel = TransactionIsolation.READ_COMMITTED;

Review Comment:
   why do we keep the txn isolation level here? txn propagation is set to Required in RetryHandler method directly?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309018298


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1804,263 +572,56 @@ protected void updateWSCommitIdAndCleanUpMetadata(Statement stmt, long txnid, Tx
     }
   }
 
-  private Optional<CompactionInfo> getCompactionByTxnId(Connection dbConn, long txnid) throws SQLException, MetaException {
-    CompactionInfo info = null;
-    try (PreparedStatement pStmt = dbConn.prepareStatement(SELECT_COMPACTION_QUEUE_BY_TXN_ID)) {
-      pStmt.setLong(1, txnid);
-      try (ResultSet rs = pStmt.executeQuery()) {
-        if (rs.next()) {
-          info = CompactionInfo.loadFullFromCompactionQueue(rs);
-        }
-      }
-    }
-    return Optional.ofNullable(info);
-  }
-
   @Override
   public Optional<CompactionInfo> getCompactionByTxnId(long txnId) throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        return getCompactionByTxnId(dbConn, txnId);
-      } catch (SQLException e) {
-        LOG.error("Unable to getCompactionByTxnId", e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "getCompactionByTxnId");
-        throw new MetaException("Unable to execute getCompactionByTxnId() " + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      return getCompactionByTxnId(txnId);
-    }
+    return Optional.ofNullable(getCompactionByTxnId(jdbcTemplate, txnId));
+  }
+
+  private CompactionInfo getCompactionByTxnId(DataSourceWrapper dataSourceWrapper, long txnId) throws MetaException {

Review Comment:
   why do we need extra private method?
   ````
   return Optional.ofNullable(rez);
   ````



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309787885


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;
+  }
+
+  /**
+   * @return Returns true if the current {@link Thread} has an associated {@link RetryContext}, false otherwise.
+   */
+  boolean hasRetryContext() {
+    return threadLocal.get() != null;
+  }
+
+  /**
+   * @return Returns the {@link NamedParameterJdbcTemplate} associated with the current {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public NamedParameterJdbcTemplate getJdbcTemplate() {
+    return jdbcTemplates.get(RetryContext.getRetryContext().datasource);
+  }
+
+  /**
+   * Gets a connection to the {@link DataSource} associated with the current {@link RetryContext}. Ensures that the same 
+   * instance is returned all the time within a particular transaction. Can be called only within an established 
+   * {@link RetryContext}. 
+   * @return Returns a connection to the datasource.
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public Connection getConnection() {
+    return DataSourceUtils.getConnection(dataSources.get(RetryContext.getRetryContext().datasource));
+  }
+
+  /**
+   * @return Returns a {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DatabaseProduct getDatabaseProduct() {
+    return databaseProduct;
+  }
+
+  /**
+   * <p>Establishes a {@link RetryContext} on the current {@link Thread}: Creates a new transaction with the given 
+   * {@link TransactionDefinition} using an internal {@link PlatformTransactionManager} associated with the reffered 
+   * {@link DataSource}. Returns a {@link TransactionStatus} representing the created transaction. The returned 
+   * {@link TransactionStatus} instance can be used for manual transaction handling.</p> 
+   * <br>
+   * <p>!!!! Please note that if there already is a {@link RetryContext} established, this call will create a nested</p>
+   * {@link RetryContext} (and transaction).</b>  
+   * @param definition The {@link TransactionDefinition} to use for creating a new transaction.
+   * @param dataSource The identifier of the {@link DataSource} to use
+   * @return Returns a {@link TransactionStatus} object representing the created transaction.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#getTransaction(TransactionDefinition)}
+   */
+  public TransactionStatus getTransactionWithinRetryContext(TransactionDefinition definition, String dataSource) throws TransactionException {
+    TransactionStatus status = transactionManagers.get(dataSource).getTransaction(definition);
+    RetryContext.setRetryContext(status, dataSource);
+    return status;
+  }
+
+  /**
+   * Commits the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#commit(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void commit() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();
+      transactionManagers.get(context.datasource).commit(context.transactionStatus);
+    } finally {
+      RetryContext.clearRetryContext();
+    }
+  }
+
+  /**
+   * Rollbacks the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#rollback(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void rollback() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();
+      transactionManagers.get(context.datasource).rollback(context.transactionStatus);
+    } finally {
+      RetryContext.clearRetryContext();
+    }
+  }
+
+  /**
+   * Executes a {@link NamedParameterJdbcTemplate#update(String, org.springframework.jdbc.core.namedparam.SqlParameterSource)}
+   * calls using the query string and parameters obtained from {@link ParameterizedCommand#getParameterizedQueryString(DatabaseProduct)} and
+   * {@link ParameterizedCommand#getQueryParameters()} methods. Validates the resulted number of affected rows using the
+   * {@link ParameterizedCommand#resultPolicy()} function.
+   * @param command The {@link ParameterizedCommand} to execute.
+   * @return Returns the number of affected rows.
+   * @throws MetaException Forwarded from {@link ParameterizedCommand#getParameterizedQueryString(DatabaseProduct)} or
+   *                       thrown if the update count was rejected by the {@link ParameterizedCommand#resultPolicy()} method
+   */
+  public Integer execute(ParameterizedCommand command) throws MetaException {
+    return execute(command.getParameterizedQueryString(getDatabaseProduct()),
+        command.getQueryParameters(), command.resultPolicy());
+  }
+
+  /**
+   * Executes a {@link NamedParameterJdbcTemplate#update(String, org.springframework.jdbc.core.namedparam.SqlParameterSource)}
+   * calls using the query string and {@link SqlParameterSource}. Validates the resulted number of affected rows using the
+   * resultpolicy function.
+   * @param query Parameterized query string.
+   * @param params Qyery parameters
+   * @param resultPolicy Result policy to use, or null, if no result policy.
+   * @return Returns the number of affected rows.
+   * @throws MetaException Forwarded from {@link ParameterizedCommand#getParameterizedQueryString(DatabaseProduct)} or
+   *                       thrown if the update count was rejected by the {@link ParameterizedCommand#resultPolicy()} method
+   */
+  public Integer execute(String query, SqlParameterSource params,
+                                Function<Integer, Boolean> resultPolicy) throws MetaException {
+    LOG.debug("Going to execute command <{}>", query);
+    int count = getJdbcTemplate().update(query, params);
+    if (resultPolicy != null && !resultPolicy.apply(count)) {
+      LOG.error("The update count was " + count + " which is not the expected. Rolling back.");
+      throw new MetaException("The update count was " + count + " which is not the expected. Rolling back.");
+    }
+    LOG.debug("Command <{}> updated {} records.", query, count);
+    return count;
+  }
+
+  /**
+   * Executes a {@link NamedParameterJdbcTemplate#query(String, SqlParameterSource, ResultSetExtractor)} call using the query 
+   * string and parameters obtained from {@link QueryHandler#getParameterizedQueryString(DatabaseProduct)} and 
+   * {@link QueryHandler#getQueryParameters()} methods. Processes the result using the {@link QueryHandler#extractData(ResultSet)}
+   * method ({@link QueryHandler} extends the {@link ResultSetExtractor} interface).
+   * @param queryHandler The {@link QueryHandler} instance containing the query, {@link SqlParameterSource}, and {@link ResultSetExtractor}.
+   * @return Returns with the object(s) constructed from the result of the executed query. 
+   * @throws MetaException Forwarded from {@link ParameterizedCommand#getParameterizedQueryString(DatabaseProduct)}.
+   */
+  public <Result> Result execute(QueryHandler<Result> queryHandler) throws MetaException {
+    String queryStr = queryHandler.getParameterizedQueryString(getDatabaseProduct());
+    LOG.debug("Going to execute query <{}>", queryStr);
+    SqlParameterSource params = queryHandler.getQueryParameters();
+    if (params != null) {
+      return getJdbcTemplate().query(queryStr, params, queryHandler);
+    } else {
+      return getJdbcTemplate().query(queryStr, queryHandler);
+    }
+  }
+
+  /**
+   * Represents a transaciton associated with a {@link DataSource} and a {@link Thread}. Holds the {@link TransactionStatus}
+   * representing the transaction, and the name of the {@link DataSource} used to create the transaction. Used for two things:
+   * <ul>
+   *   <li>
+   *     Its presence tells to {@link RetryHandler} that there is no need for new (nested) context, it can use the
+   *     existing one
+   *   </li>
+   *   <li>
+   *     The {@link DataSourceWrapper#getTransactionStatus()}, {@link DataSourceWrapper#getConnection()}, 
+   *     {@link DataSourceWrapper#getJdbcTemplate()}, {@link DataSourceWrapper#commit()}, {@link DataSourceWrapper#rollback()}
+   *     methods are using it to identify the correct datasource which is associated with the current
+   *     {@link RetryContext} (and {@link Thread}).
+   *   </li>
+   * </ul>
+   */
+  static class RetryContext {
+
+    /**
+     * Retrurns the previously established {@link RetryContext}. Can be called only within an established {@link RetryContext}.
+     * @return Retruns the previously established {@link RetryContext}.
+     * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+     */
+    private static RetryContext getRetryContext() {
+      RetryContext context = threadLocal.get();
+      if (context == null) {
+        throw new IllegalStateException("Trying to access a transactional resource without retry context!");
+      }
+      return context;
+    }
+    
+    /**
+     * Establishes a {@link RetryContext} on the current thread using the given {@link TransactionStatus}. 
+     * @param status The {@link TransactionStatus} to bind to the current {@link Thread}.
+     * @param datasource The name of the {@link DataSource} used to create the {@link TransactionStatus}.
+     */
+    private static void setRetryContext(TransactionStatus status, String datasource) {
+      RetryContext old = threadLocal.get();
+      threadLocal.set(new RetryContext(status, datasource, old));
+    }
+
+    /**
+     * Clears a previously established {@link RetryContext} on the current {@link Thread}.
+     */
+    private static void clearRetryContext() {
+      RetryContext top = threadLocal.get();
+      if(top != null && top.parent != null) {

Review Comment:
   nit: space



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309959673


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2575,11 +2601,17 @@ protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException,
           minOpenTxn = Long.MAX_VALUE;
         }
       }
+    } catch (SQLException e) {
+      throw new UncategorizedSQLException(null, null, e);
+    }
+    jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);
+    try {
+      long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+      LOG.debug("MinOpenTxnIdWaterMark calculated with minOpenTxn {}, lowWaterMark {}", minOpenTxn, lowWaterMark);
+      return Long.min(minOpenTxn, lowWaterMark + 1);
+    } finally {
+      jdbcTemplate.commit();

Review Comment:
   yes



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310003447


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));
+    }
+    this.databaseProduct = databaseProduct;
+  }
+
+
+  /**
+   * Provides direct access to the {@link TransactionStatus} to allow manual savepoint creation, rollback, etc.
+   * Can be called only within an established {@link RetryContext}.
+   * @return Returns the {@link TransactionStatus} instance stored in the {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public TransactionStatus getTransactionStatus() {
+    return RetryContext.getRetryContext().transactionStatus;
+  }
+
+  /**
+   * @return Returns true if the current {@link Thread} has an associated {@link RetryContext}, false otherwise.
+   */
+  boolean hasRetryContext() {
+    return threadLocal.get() != null;
+  }
+
+  /**
+   * @return Returns the {@link NamedParameterJdbcTemplate} associated with the current {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws IllegalStateException Thrown when called outside a {@link RetryContext}.
+   */
+  public NamedParameterJdbcTemplate getJdbcTemplate() {
+    return jdbcTemplates.get(RetryContext.getRetryContext().datasource);
+  }
+
+  /**
+   * Gets a connection to the {@link DataSource} associated with the current {@link RetryContext}. Ensures that the same 
+   * instance is returned all the time within a particular transaction. Can be called only within an established 
+   * {@link RetryContext}. 
+   * @return Returns a connection to the datasource.
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public Connection getConnection() {
+    return DataSourceUtils.getConnection(dataSources.get(RetryContext.getRetryContext().datasource));
+  }
+
+  /**
+   * @return Returns a {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DatabaseProduct getDatabaseProduct() {
+    return databaseProduct;
+  }
+
+  /**
+   * <p>Establishes a {@link RetryContext} on the current {@link Thread}: Creates a new transaction with the given 
+   * {@link TransactionDefinition} using an internal {@link PlatformTransactionManager} associated with the reffered 
+   * {@link DataSource}. Returns a {@link TransactionStatus} representing the created transaction. The returned 
+   * {@link TransactionStatus} instance can be used for manual transaction handling.</p> 
+   * <br>
+   * <p>!!!! Please note that if there already is a {@link RetryContext} established, this call will create a nested</p>
+   * {@link RetryContext} (and transaction).</b>  
+   * @param definition The {@link TransactionDefinition} to use for creating a new transaction.
+   * @param dataSource The identifier of the {@link DataSource} to use
+   * @return Returns a {@link TransactionStatus} object representing the created transaction.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#getTransaction(TransactionDefinition)}
+   */
+  public TransactionStatus getTransactionWithinRetryContext(TransactionDefinition definition, String dataSource) throws TransactionException {
+    TransactionStatus status = transactionManagers.get(dataSource).getTransaction(definition);
+    RetryContext.setRetryContext(status, dataSource);
+    return status;
+  }
+
+  /**
+   * Commits the transaction associated with the current {@link Thread} and clears the {@link RetryContext}.
+   * Can be called only within an established {@link RetryContext}.
+   * @throws TransactionException Forwarded from {@link PlatformTransactionManager#commit(TransactionStatus)}
+   * @throws IllegalArgumentException Thrown when called outside a {@link RetryContext}
+   */
+  public void commit() throws TransactionException {
+    try {
+      RetryContext context = RetryContext.getRetryContext();
+      transactionManagers.get(context.datasource).commit(context.transactionStatus);

Review Comment:
   The context is also required to obtain the txn manager `(transactionManagers.get(context.datasource)`, so using the helper method would lead to querying the `RetryContect` twice.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309932393


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1804,263 +572,56 @@ protected void updateWSCommitIdAndCleanUpMetadata(Statement stmt, long txnid, Tx
     }
   }
 
-  private Optional<CompactionInfo> getCompactionByTxnId(Connection dbConn, long txnid) throws SQLException, MetaException {
-    CompactionInfo info = null;
-    try (PreparedStatement pStmt = dbConn.prepareStatement(SELECT_COMPACTION_QUEUE_BY_TXN_ID)) {
-      pStmt.setLong(1, txnid);
-      try (ResultSet rs = pStmt.executeQuery()) {
-        if (rs.next()) {
-          info = CompactionInfo.loadFullFromCompactionQueue(rs);
-        }
-      }
-    }
-    return Optional.ofNullable(info);
-  }
-
   @Override
   public Optional<CompactionInfo> getCompactionByTxnId(long txnId) throws MetaException {
-    Connection dbConn = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        return getCompactionByTxnId(dbConn, txnId);
-      } catch (SQLException e) {
-        LOG.error("Unable to getCompactionByTxnId", e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "getCompactionByTxnId");
-        throw new MetaException("Unable to execute getCompactionByTxnId() " + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      return getCompactionByTxnId(txnId);
-    }
+    return Optional.ofNullable(getCompactionByTxnId(jdbcTemplate, txnId));
+  }
+
+  private CompactionInfo getCompactionByTxnId(DataSourceWrapper dataSourceWrapper, long txnId) throws MetaException {

Review Comment:
   Good point, that method is a remnant of an earlier approach where it was needed. Removed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310035924


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,38 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).execute(jdbcTemplate);

Review Comment:
   1st: While it was a great idea for QueryHandlers and ParameterizedCommands, for the TransactionalFunction it would look like: 
   ```
     public <R> R execute(TransactionalFunction<R> function) throws SQLException, MetaException {
       return function.execute(this);
     }
   
   ```
   In this case it doesn't have much added value.
   
   2st: fixed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1336992589


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +158,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    jdbcResource.execute(
+        "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = :state, \"CQ_WORKER_ID\" = NULL WHERE \"CQ_ID\" = :id",

Review Comment:
   should it be `IS NULL` instead of `= NULL`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288098969


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java:
##########
@@ -385,16 +384,14 @@ private boolean foundCurrentOrFailedCompactions(ShowCompactResponse compactions,
       LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " +
         MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed.");
 
-      ci.errorMessage = "Compaction is not initiated since last " +
-        MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " consecutive compaction attempts failed)";
-
-      txnHandler.markFailed(ci);
+      txnHandler.markFailed(ci.toFailedCompacion("Compaction is not initiated since last " +
+          MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " consecutive compaction attempts failed)"));
       return true;
     }
     return false;
   }
 
-  private CompactionType checkForCompaction(final CompactionInfo ci,
+  private CompactionRequest checkForCompaction(final CompactionCandidate ci,

Review Comment:
   that is confusing method naming. checkForCompaction answers what type of compaction is needed if any, but doesn't generate the compaction request. request should be created only if compaction is required
   ````
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1285516119


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfoBase.java:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+public class CompactionInfoBase implements Comparable<CompactionInfoBase> {

Review Comment:
   why do we need this? can't we just use 1 CompactionInfo object? 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1285848071


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfoBase.java:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+public class CompactionInfoBase implements Comparable<CompactionInfoBase> {

Review Comment:
   Currently CompactionInfo is like an union of all compaction related information. In numerous cases only a fraction of the fields are loaded to save resources (`findNextToCompact`, `findReadyToClean`), and in some other cases like `findPotentialCompactions` in Initiator, several fields are not even applicable for the scenario. This results in a situation that Compaction info is almost never fully initialized and you have to be aware of the context where it is used: You need to know which fields are present and which not. While technically this is a fully working approach, I would prefer introduce some class hierarchy. For example `CompactionCandidate` is the entity used by Initiator. Having this object makes it much more explicit which information is used/available in the initialization phase, and which others are not yet present/applicable. This is the idea behind the introduction of these two new base classes.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288398247


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java:
##########
@@ -385,16 +384,14 @@ private boolean foundCurrentOrFailedCompactions(ShowCompactResponse compactions,
       LOG.warn("Will not initiate compaction for " + ci.getFullPartitionName() + " since last " +
         MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " attempts to compact it failed.");
 
-      ci.errorMessage = "Compaction is not initiated since last " +
-        MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " consecutive compaction attempts failed)";
-
-      txnHandler.markFailed(ci);
+      txnHandler.markFailed(ci.toFailedCompacion("Compaction is not initiated since last " +
+          MetastoreConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD + " consecutive compaction attempts failed)"));
       return true;
     }
     return false;
   }
 
-  private CompactionType checkForCompaction(final CompactionInfo ci,
+  private CompactionRequest checkForCompaction(final CompactionCandidate ci,

Review Comment:
   Removed all CompactionInfo class structure related changes



##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java:
##########
@@ -445,17 +443,34 @@ private CompactionType checkForCompaction(final CompactionInfo ci,
               ci.getFullPartitionName(), exception);
         }
       }
-      return compactionType;
+    }
+    if (compactionType != null) {
+      return createCompactionRequest(ci, compactionType);
+    } else {
+      return null;
     }
   }
+  
+  private CompactionRequest createCompactionRequest(CompactionCandidate ci, CompactionType type) {

Review Comment:
   Removed all CompactionInfo class structure related changes



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293066381


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1396,245 +388,59 @@ private void insertAbortRetryRetentionTimeOnError(Connection dbConn, CompactionI
   @Override
   @RetrySemantics.SafeToRetry
   public void purgeCompactionHistory() throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    List<Long> deleteSet = new ArrayList<>();
-    RetentionCounters rc = null;
-    long timeoutThreshold = System.currentTimeMillis() -
-            MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_TIMEOUT, TimeUnit.MILLISECONDS);
-    int didNotInitiateRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE);
-    int failedRetention = getFailedCompactionRetention();
-    int succeededRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_SUCCEEDED);
-    int refusedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_REFUSED);
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        /* cc_id is monotonically increasing so for any entity sorts in order of compaction history,
-        thus this query groups by entity and withing group sorts most recent first */
-        rs = stmt.executeQuery("SELECT \"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", "
-            + "\"CC_STATE\" , \"CC_START\", \"CC_TYPE\" "
-            + "FROM \"COMPLETED_COMPACTIONS\" ORDER BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"," +
-                "\"CC_ID\" DESC");
-        String lastCompactedEntity = null;
-        /* In each group, walk from most recent and count occurrences of each state type.  Once you
-        * have counted enough (for each state) to satisfy retention policy, delete all other
-        * instances of this status, plus timed-out entries (see this method's JavaDoc).
-        */
-        while(rs.next()) {
-          CompactionInfo ci = new CompactionInfo(
-              rs.getLong(1), rs.getString(2), rs.getString(3),
-              rs.getString(4), rs.getString(5).charAt(0));
-          ci.start = rs.getLong(6);
-          ci.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(7).charAt(0));
-          if(!ci.getFullPartitionName().equals(lastCompactedEntity)) {
-            lastCompactedEntity = ci.getFullPartitionName();
-            rc = new RetentionCounters(didNotInitiateRetention, failedRetention, succeededRetention, refusedRetention);
-          }
-          checkForDeletion(deleteSet, ci, rc, timeoutThreshold);
-        }
-        close(rs);
-
-        if (deleteSet.size() <= 0) {
-          return;
-        }
-
-        List<String> queries = new ArrayList<>();
-
-        StringBuilder prefix = new StringBuilder();
-        StringBuilder suffix = new StringBuilder();
-
-        prefix.append("DELETE FROM \"COMPLETED_COMPACTIONS\" WHERE ");
-
-        List<String> questions = new ArrayList<>(deleteSet.size());
-        for (int  i = 0; i < deleteSet.size(); i++) {
-          questions.add("?");
-        }
-        List<Integer> counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix, suffix, questions,
-            "\"CC_ID\"", false, false);
-        int totalCount = 0;
-        for (int i = 0; i < queries.size(); i++) {
-          String query = queries.get(i);
-          long insertCount = counts.get(i);
-          LOG.debug("Going to execute update <{}>", query);
-          pStmt = dbConn.prepareStatement(query);
-          for (int j = 0; j < insertCount; j++) {
-            pStmt.setLong(j + 1, deleteSet.get(totalCount + j));
-          }
-          totalCount += insertCount;
-          int count = pStmt.executeUpdate();
-          LOG.debug("Removed {} records from COMPLETED_COMPACTIONS", count);
-        }
-        dbConn.commit();
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "purgeCompactionHistory()");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-        closeStmt(pStmt);
-      }
-    } catch (RetryException ex) {
-      purgeCompactionHistory();
-    }
-  }
-
-  /**
-   * this ensures that the number of failed compaction entries retained is > than number of failed
-   * compaction threshold which prevents new compactions from being scheduled.
-   */
-  private int getFailedCompactionRetention() {
-    int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-    int failedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED);
-    if(failedRetention < failedThreshold) {
-      LOG.warn("Invalid configuration {}={} < {}={}.  Will use {}={}",
-          ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.getVarname(), failedRetention,
-          ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED, failedRetention,
-          ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.getVarname(), failedRetention);
-      failedRetention = failedThreshold;
-    }
-    return failedRetention;
+    new PurgeCompactionHistoryFunction(conf).call(dataSourceWrapper);
   }
 
   /**
    * Returns {@code true} if there already exists sufficient number of consecutive failures for
    * this table/partition so that no new automatic compactions will be scheduled.
    * User initiated compactions don't do this check.
-   *
    * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
    * That would be a meta operations, i.e. first find all partitions for this table (which have
    * txn info) and schedule each compaction separately.  This avoids complications in this logic.
    */
   @Override
   @RetrySemantics.ReadOnly
   public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
-    Connection dbConn = null;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        pStmt = dbConn.prepareStatement("SELECT \"CC_STATE\", \"CC_ENQUEUE_TIME\" FROM \"COMPLETED_COMPACTIONS\" WHERE " +
-          "\"CC_DATABASE\" = ? AND " +
-          "\"CC_TABLE\" = ? " +
-          (ci.partName != null ? "AND \"CC_PARTITION\" = ?" : "") +
-          " AND \"CC_STATE\" != " + quoteChar(DID_NOT_INITIATE) + " ORDER BY \"CC_ID\" DESC");
-        pStmt.setString(1, ci.dbname);
-        pStmt.setString(2, ci.tableName);
-        if (ci.partName != null) {
-          pStmt.setString(3, ci.partName);
-        }
-        rs = pStmt.executeQuery();
-        int numFailed = 0;
-        int numTotal = 0;
-        long lastEnqueueTime = -1;
-        int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-        while(rs.next() && ++numTotal <= failedThreshold) {
-          long enqueueTime = rs.getLong(2);
-          if (enqueueTime > lastEnqueueTime) {
-            lastEnqueueTime = enqueueTime;
-          }
-          if(rs.getString(1).charAt(0) == FAILED_STATE) {
-            numFailed++;
-          }
-          else {
-            numFailed--;
-          }
-        }
-        // If the last attempt was too long ago, ignore the failed threshold and try compaction again
-        long retryTime = MetastoreConf.getTimeVar(conf,
-            ConfVars.COMPACTOR_INITIATOR_FAILED_RETRY_TIME, TimeUnit.MILLISECONDS);
-        boolean needsRetry = (retryTime > 0) && (lastEnqueueTime + retryTime < System.currentTimeMillis());
-        return (numFailed == failedThreshold) && !needsRetry;
-      }
-      catch (SQLException e) {
-        LOG.error("Unable to check for failed compactions", e);
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "checkFailedCompactions(" + ci + ")");
-        LOG.error(DB_FAILED_TO_CONNECT, e);
-        return false;//weren't able to check
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return checkFailedCompactions(ci);
-    }
+    return new CheckFailedCompactionsHandler(conf, ci).execute(dataSourceWrapper);
   }
 
-
   private void updateStatus(CompactionInfo ci) throws MetaException {
     String strState = CompactionState.fromSqlConst(ci.state).toString();
+
     LOG.debug("Marking as {}: CompactionInfo: {}", strState, ci);
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        pStmt = dbConn.prepareStatement("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-                + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-                + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-                + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-                + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-                + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?");
-        pStmt.setLong(1, ci.id);
-        rs = pStmt.executeQuery();
-        if (rs.next()) {
-          //preserve errorMessage and state
-          String errorMessage = ci.errorMessage;
-          char state = ci.state;
-          ci = CompactionInfo.loadFullFromCompactionQueue(rs);
-          ci.errorMessage = errorMessage;
-          ci.state = state;
+    CompactionInfo ciActual = new GetCompactionInfoHandler(ci.id, false).execute(dataSourceWrapper); 
 
-          pStmt = dbConn.prepareStatement(DELETE_CQ_ENTRIES);
-          pStmt.setLong(1, ci.id);
-          LOG.debug("Going to execute update <{}>", DELETE_CQ_ENTRIES);
-          pStmt.executeUpdate();
-        }
-        else {
-          if(ci.id > 0) {
-            //the record with valid CQ_ID has disappeared - this is a sign of something wrong
-            throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
-          }
-        }
-        if(ci.id == 0) {
-          //The failure occurred before we even made an entry in COMPACTION_QUEUE
-          //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
-          ci.id = generateCompactionQueueId(stmt);
-          //this is not strictly accurate, but 'type' cannot be null.
-          if(ci.type == null) {
-            ci.type = CompactionType.MINOR;
-          }
-          ci.start = getDbTime(dbConn);
-          LOG.debug("The failure occurred before we even made an entry in COMPACTION_QUEUE. Generated ID so that we "
-                  + "can make an entry in COMPLETED_COMPACTIONS. New Id: {}", ci.id);
-        }
-        close(rs, stmt, null);
-        closeStmt(pStmt);
+    long endTime = getDbTime().getTime();
+    if (ciActual != null) {
+      //preserve errorMessage and state
+      ciActual.errorMessage = ci.errorMessage;
+      ciActual.state = ci.state;
 
-        pStmt = dbConn.prepareStatement(TxnQueries.INSERT_INTO_COMPLETED_COMPACTION);
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        LOG.debug("Inserted {} entries into COMPLETED_COMPACTIONS", updCount);
-        closeStmt(pStmt);
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Failed to mark compaction request as " + strState + ", rolling back transaction: " + ci, e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "updateStatus(" + ci + ")");
-      } finally {
-        close(rs, stmt, null);
-        close(null, pStmt, dbConn);
+      dataSourceWrapper.getJdbcTemplate().update("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id",
+          new MapSqlParameterSource("id", ci.id));
+    } else {
+      if (ci.id > 0) {
+        //the record with valid CQ_ID has disappeared - this is a sign of something wrong
+        throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
+      }
+      ciActual = ci;
+    }
+    if (ciActual.id == 0) {
+      //The failure occurred before we even made an entry in COMPACTION_QUEUE
+      //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
+      ciActual.id = generateCompactionQueueId();
+      //this is not strictly accurate, but 'type' cannot be null.
+      if (ciActual.type == null) {
+        ciActual.type = CompactionType.MINOR;
       }
-    } catch (RetryException e) {
-      updateStatus(ci);
+      //in case of creating a new entry start and end time will be the same
+      ciActual.start = endTime;
+      LOG.debug("The failure occurred before we even made an entry in COMPACTION_QUEUE. Generated ID so that we "
+          + "can make an entry in COMPLETED_COMPACTIONS. New Id: {}", ciActual.id);
     }
+
+    new InsertCompactionInfoCommand(ciActual, endTime).execute(dataSourceWrapper);

Review Comment:
   not consistent naming, in some places handler in other command



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293071008


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +141,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    ParameterizedCommand.execute(dataSourceWrapper, 

Review Comment:
   i like this style



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293083680


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,39 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).call(dataSourceWrapper);

Review Comment:
   This is the third option: You can choose to derive directly from TransactionalVoidFunction or TransactionalFunction interfaces. It offers a "lower" layer as you can write not just a query/command, but anything you need.
   The naming is consistent, as it shows from which superclass the class is derived from.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293079391


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,39 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).call(dataSourceWrapper);

Review Comment:
   can we stick with common naming



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293079646


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -179,90 +93,12 @@ public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold, long a
   @RetrySemantics.ReadOnly
   public Set<CompactionInfo> findPotentialCompactions(int abortedThreshold,
       long abortedTimeThreshold, long lastChecked) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        long startedAt = System.currentTimeMillis();
-        long checkInterval = (lastChecked <= 0) ? lastChecked : (startedAt - lastChecked + 500) / 1000;
-
-        // Check for completed transactions
-        final String s = dbProduct.addLimitClause(
-          MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-          "DISTINCT \"TC\".\"CTC_DATABASE\", \"TC\".\"CTC_TABLE\", \"TC\".\"CTC_PARTITION\" " +
-          "FROM \"COMPLETED_TXN_COMPONENTS\" \"TC\" " + (checkInterval > 0 ?
-          "LEFT JOIN ( " +
-          "  SELECT \"C1\".* FROM \"COMPLETED_COMPACTIONS\" \"C1\" " +
-          "  INNER JOIN ( " +
-          "    SELECT MAX(\"CC_ID\") \"CC_ID\" FROM \"COMPLETED_COMPACTIONS\" " +
-          "    GROUP BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"" +
-          "  ) \"C2\" " +
-          "  ON \"C1\".\"CC_ID\" = \"C2\".\"CC_ID\" " +
-          "  WHERE \"C1\".\"CC_STATE\" IN (" + quoteChar(DID_NOT_INITIATE) + "," + quoteChar(FAILED_STATE) + ")" +
-          ") \"C\" " +
-          "ON \"TC\".\"CTC_DATABASE\" = \"C\".\"CC_DATABASE\" AND \"TC\".\"CTC_TABLE\" = \"C\".\"CC_TABLE\" " +
-          "  AND (\"TC\".\"CTC_PARTITION\" = \"C\".\"CC_PARTITION\" OR (\"TC\".\"CTC_PARTITION\" IS NULL AND \"C\".\"CC_PARTITION\" IS NULL)) " +
-          "WHERE \"C\".\"CC_ID\" IS NOT NULL OR " + isWithinCheckInterval("\"TC\".\"CTC_TIMESTAMP\"", checkInterval) : ""));
-
-        LOG.debug("Going to execute query <{}>", s);
-
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        if (!MetastoreConf.getBoolVar(conf, ConfVars.COMPACTOR_CLEAN_ABORTS_USING_CLEANER)) {
-          // Check for aborted txns: number of aborted txns past threshold and age of aborted txns
-          // past time threshold
-          boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-          String sCheckAborted = "SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", " +
-                  "MIN(\"TXN_STARTED\"), COUNT(*) FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-                  "   WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED + " " +
-                  "GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" " +
-                  (checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold);
-
-          LOG.debug("Going to execute query <{}>", sCheckAborted);
-          rs = stmt.executeQuery(sCheckAborted);
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              info.tooManyAborts = numAbortedTxns > abortedThreshold;
-              info.hasOldAbort = pastTimeThreshold;
-              LOG.debug("Found potential compaction: {}", info);
-              response.add(info);
-            }
-          }
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e,
-            "findPotentialCompactions(maxAborted:" + abortedThreshold
-                + ", abortedTimeThreshold:" + abortedTimeThreshold + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(abortedThreshold, abortedTimeThreshold, lastChecked);
+    Set<CompactionInfo> candidates = new HashSet<>(new CompactionCandidateHandler(lastChecked, 

Review Comment:
   I was already thinking in sth like that. The problem is that the parameters are varying in every QueryHandler implementation, so it could be done only if the parameters are passed as an Object array. I was also thinking of passing a prepared SqlParameterSource, but some handlers are requiring other parameters, like conf variables, or modifier flags, and those parameters are not used in the SQL itself. So I decided to pass the parameters in the constructor, so the differences are not affecting the interface method. If you have any other ideas please share it.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1582811977

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [34 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1613926092

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [67 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288082568


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfoBase.java:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+public class CompactionInfoBase implements Comparable<CompactionInfoBase> {

Review Comment:
   could we do this in a separate PR and focus here just on CompactionTxnHandler?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288104122


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java:
##########
@@ -445,17 +443,34 @@ private CompactionType checkForCompaction(final CompactionInfo ci,
               ci.getFullPartitionName(), exception);
         }
       }
-      return compactionType;
+    }
+    if (compactionType != null) {
+      return createCompactionRequest(ci, compactionType);
+    } else {
+      return null;
     }
   }
+  
+  private CompactionRequest createCompactionRequest(CompactionCandidate ci, CompactionType type) {

Review Comment:
   i don't like this refactor, since it doesn't create a complete CompactionRequest and does some initialization afterward. in `requestCompaction ` you can see the complete request



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1288397693


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionInfoBase.java:
##########
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+public class CompactionInfoBase implements Comparable<CompactionInfoBase> {

Review Comment:
   Removed all CompactionInfo class structure related changes



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entities/CompactionCandidate.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entities;
+
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+
+public class CompactionCandidate extends CompactionInfoBase {
+
+  public boolean tooManyAborts;
+  public boolean hasOldAbort;
+  
+  public CompactionInfo toFailedCompacion(String errorMessage) {

Review Comment:
   Removed all CompactionInfo class structure related changes



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293073769


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1396,245 +388,59 @@ private void insertAbortRetryRetentionTimeOnError(Connection dbConn, CompactionI
   @Override
   @RetrySemantics.SafeToRetry
   public void purgeCompactionHistory() throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    List<Long> deleteSet = new ArrayList<>();
-    RetentionCounters rc = null;
-    long timeoutThreshold = System.currentTimeMillis() -
-            MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_TIMEOUT, TimeUnit.MILLISECONDS);
-    int didNotInitiateRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE);
-    int failedRetention = getFailedCompactionRetention();
-    int succeededRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_SUCCEEDED);
-    int refusedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_REFUSED);
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        /* cc_id is monotonically increasing so for any entity sorts in order of compaction history,
-        thus this query groups by entity and withing group sorts most recent first */
-        rs = stmt.executeQuery("SELECT \"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", "
-            + "\"CC_STATE\" , \"CC_START\", \"CC_TYPE\" "
-            + "FROM \"COMPLETED_COMPACTIONS\" ORDER BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"," +
-                "\"CC_ID\" DESC");
-        String lastCompactedEntity = null;
-        /* In each group, walk from most recent and count occurrences of each state type.  Once you
-        * have counted enough (for each state) to satisfy retention policy, delete all other
-        * instances of this status, plus timed-out entries (see this method's JavaDoc).
-        */
-        while(rs.next()) {
-          CompactionInfo ci = new CompactionInfo(
-              rs.getLong(1), rs.getString(2), rs.getString(3),
-              rs.getString(4), rs.getString(5).charAt(0));
-          ci.start = rs.getLong(6);
-          ci.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(7).charAt(0));
-          if(!ci.getFullPartitionName().equals(lastCompactedEntity)) {
-            lastCompactedEntity = ci.getFullPartitionName();
-            rc = new RetentionCounters(didNotInitiateRetention, failedRetention, succeededRetention, refusedRetention);
-          }
-          checkForDeletion(deleteSet, ci, rc, timeoutThreshold);
-        }
-        close(rs);
-
-        if (deleteSet.size() <= 0) {
-          return;
-        }
-
-        List<String> queries = new ArrayList<>();
-
-        StringBuilder prefix = new StringBuilder();
-        StringBuilder suffix = new StringBuilder();
-
-        prefix.append("DELETE FROM \"COMPLETED_COMPACTIONS\" WHERE ");
-
-        List<String> questions = new ArrayList<>(deleteSet.size());
-        for (int  i = 0; i < deleteSet.size(); i++) {
-          questions.add("?");
-        }
-        List<Integer> counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix, suffix, questions,
-            "\"CC_ID\"", false, false);
-        int totalCount = 0;
-        for (int i = 0; i < queries.size(); i++) {
-          String query = queries.get(i);
-          long insertCount = counts.get(i);
-          LOG.debug("Going to execute update <{}>", query);
-          pStmt = dbConn.prepareStatement(query);
-          for (int j = 0; j < insertCount; j++) {
-            pStmt.setLong(j + 1, deleteSet.get(totalCount + j));
-          }
-          totalCount += insertCount;
-          int count = pStmt.executeUpdate();
-          LOG.debug("Removed {} records from COMPLETED_COMPACTIONS", count);
-        }
-        dbConn.commit();
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "purgeCompactionHistory()");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-        closeStmt(pStmt);
-      }
-    } catch (RetryException ex) {
-      purgeCompactionHistory();
-    }
-  }
-
-  /**
-   * this ensures that the number of failed compaction entries retained is > than number of failed
-   * compaction threshold which prevents new compactions from being scheduled.
-   */
-  private int getFailedCompactionRetention() {
-    int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-    int failedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED);
-    if(failedRetention < failedThreshold) {
-      LOG.warn("Invalid configuration {}={} < {}={}.  Will use {}={}",
-          ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.getVarname(), failedRetention,
-          ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED, failedRetention,
-          ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.getVarname(), failedRetention);
-      failedRetention = failedThreshold;
-    }
-    return failedRetention;
+    new PurgeCompactionHistoryFunction(conf).call(dataSourceWrapper);
   }
 
   /**
    * Returns {@code true} if there already exists sufficient number of consecutive failures for
    * this table/partition so that no new automatic compactions will be scheduled.
    * User initiated compactions don't do this check.
-   *
    * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
    * That would be a meta operations, i.e. first find all partitions for this table (which have
    * txn info) and schedule each compaction separately.  This avoids complications in this logic.
    */
   @Override
   @RetrySemantics.ReadOnly
   public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
-    Connection dbConn = null;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        pStmt = dbConn.prepareStatement("SELECT \"CC_STATE\", \"CC_ENQUEUE_TIME\" FROM \"COMPLETED_COMPACTIONS\" WHERE " +
-          "\"CC_DATABASE\" = ? AND " +
-          "\"CC_TABLE\" = ? " +
-          (ci.partName != null ? "AND \"CC_PARTITION\" = ?" : "") +
-          " AND \"CC_STATE\" != " + quoteChar(DID_NOT_INITIATE) + " ORDER BY \"CC_ID\" DESC");
-        pStmt.setString(1, ci.dbname);
-        pStmt.setString(2, ci.tableName);
-        if (ci.partName != null) {
-          pStmt.setString(3, ci.partName);
-        }
-        rs = pStmt.executeQuery();
-        int numFailed = 0;
-        int numTotal = 0;
-        long lastEnqueueTime = -1;
-        int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-        while(rs.next() && ++numTotal <= failedThreshold) {
-          long enqueueTime = rs.getLong(2);
-          if (enqueueTime > lastEnqueueTime) {
-            lastEnqueueTime = enqueueTime;
-          }
-          if(rs.getString(1).charAt(0) == FAILED_STATE) {
-            numFailed++;
-          }
-          else {
-            numFailed--;
-          }
-        }
-        // If the last attempt was too long ago, ignore the failed threshold and try compaction again
-        long retryTime = MetastoreConf.getTimeVar(conf,
-            ConfVars.COMPACTOR_INITIATOR_FAILED_RETRY_TIME, TimeUnit.MILLISECONDS);
-        boolean needsRetry = (retryTime > 0) && (lastEnqueueTime + retryTime < System.currentTimeMillis());
-        return (numFailed == failedThreshold) && !needsRetry;
-      }
-      catch (SQLException e) {
-        LOG.error("Unable to check for failed compactions", e);
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "checkFailedCompactions(" + ci + ")");
-        LOG.error(DB_FAILED_TO_CONNECT, e);
-        return false;//weren't able to check
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return checkFailedCompactions(ci);
-    }
+    return new CheckFailedCompactionsHandler(conf, ci).execute(dataSourceWrapper);
   }
 
-
   private void updateStatus(CompactionInfo ci) throws MetaException {
     String strState = CompactionState.fromSqlConst(ci.state).toString();
+
     LOG.debug("Marking as {}: CompactionInfo: {}", strState, ci);
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        pStmt = dbConn.prepareStatement("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-                + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-                + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-                + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-                + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-                + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?");
-        pStmt.setLong(1, ci.id);
-        rs = pStmt.executeQuery();
-        if (rs.next()) {
-          //preserve errorMessage and state
-          String errorMessage = ci.errorMessage;
-          char state = ci.state;
-          ci = CompactionInfo.loadFullFromCompactionQueue(rs);
-          ci.errorMessage = errorMessage;
-          ci.state = state;
+    CompactionInfo ciActual = new GetCompactionInfoHandler(ci.id, false).execute(dataSourceWrapper); 
 
-          pStmt = dbConn.prepareStatement(DELETE_CQ_ENTRIES);
-          pStmt.setLong(1, ci.id);
-          LOG.debug("Going to execute update <{}>", DELETE_CQ_ENTRIES);
-          pStmt.executeUpdate();
-        }
-        else {
-          if(ci.id > 0) {
-            //the record with valid CQ_ID has disappeared - this is a sign of something wrong
-            throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
-          }
-        }
-        if(ci.id == 0) {
-          //The failure occurred before we even made an entry in COMPACTION_QUEUE
-          //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
-          ci.id = generateCompactionQueueId(stmt);
-          //this is not strictly accurate, but 'type' cannot be null.
-          if(ci.type == null) {
-            ci.type = CompactionType.MINOR;
-          }
-          ci.start = getDbTime(dbConn);
-          LOG.debug("The failure occurred before we even made an entry in COMPACTION_QUEUE. Generated ID so that we "
-                  + "can make an entry in COMPLETED_COMPACTIONS. New Id: {}", ci.id);
-        }
-        close(rs, stmt, null);
-        closeStmt(pStmt);
+    long endTime = getDbTime().getTime();
+    if (ciActual != null) {
+      //preserve errorMessage and state
+      ciActual.errorMessage = ci.errorMessage;
+      ciActual.state = ci.state;
 
-        pStmt = dbConn.prepareStatement(TxnQueries.INSERT_INTO_COMPLETED_COMPACTION);
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        LOG.debug("Inserted {} entries into COMPLETED_COMPACTIONS", updCount);
-        closeStmt(pStmt);
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Failed to mark compaction request as " + strState + ", rolling back transaction: " + ci, e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "updateStatus(" + ci + ")");
-      } finally {
-        close(rs, stmt, null);
-        close(null, pStmt, dbConn);
+      dataSourceWrapper.getJdbcTemplate().update("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id",
+          new MapSqlParameterSource("id", ci.id));
+    } else {
+      if (ci.id > 0) {
+        //the record with valid CQ_ID has disappeared - this is a sign of something wrong
+        throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
+      }
+      ciActual = ci;
+    }
+    if (ciActual.id == 0) {
+      //The failure occurred before we even made an entry in COMPACTION_QUEUE
+      //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
+      ciActual.id = generateCompactionQueueId();
+      //this is not strictly accurate, but 'type' cannot be null.
+      if (ciActual.type == null) {
+        ciActual.type = CompactionType.MINOR;
       }
-    } catch (RetryException e) {
-      updateStatus(ci);
+      //in case of creating a new entry start and end time will be the same
+      ciActual.start = endTime;
+      LOG.debug("The failure occurred before we even made an entry in COMPACTION_QUEUE. Generated ID so that we "
+          + "can make an entry in COMPLETED_COMPACTIONS. New Id: {}", ciActual.id);
     }
+
+    new InsertCompactionInfoCommand(ciActual, endTime).execute(dataSourceWrapper);

Review Comment:
   It is consistent. There are two base classes: `QueryHandler` and `ParameterizedCommand`. The former is for queries (with `ResultSet` mapping), the latter is for INSERT/UPDATE/DELETE/etc commands.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293087735


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1046,37 +290,14 @@ public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
    */
   @Override
   @RetrySemantics.Idempotent
-  public void revokeFromLocalWorkers(String hostname) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL, \"CQ_STATE\" = '"
-          + INITIATED_STATE+ "' WHERE \"CQ_STATE\" = '" + WORKING_STATE + "' AND \"CQ_WORKER_ID\" LIKE '"
-          +  hostname + "%'";
-        LOG.debug("Going to execute update <{}>", s);
-        // It isn't an error if the following returns no rows, as the local workers could have died
-        // with  nothing assigned to them.
-        int updated = stmt.executeUpdate(s);
-        LOG.debug("Set {} compaction queue entries to {} state for host {}", updated, INITIATED_RESPONSE, hostname);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to change dead worker's records back to initiated state " +
-          e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "revokeFromLocalWorkers(hostname:" + hostname +")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      revokeFromLocalWorkers(hostname);
-    }
+  public void revokeFromLocalWorkers(String hostname) {
+    dataSourceWrapper.getJdbcTemplate().update(
+        "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_WORKER_ID\" = NULL, \"CQ_START\" = NULL," +

Review Comment:
   why to extract queries from constants and do multiple strings initialization?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1713632153

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [69 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1719333925

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337121719


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/ProxyTxnHandler.java:
##########
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.TransactionContext;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.SqlRetryCallProperties;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.SqlRetry;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.SqlRetryFunction;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.SqlRetryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.Arrays;
+
+/**
+ * Responsible for processing the following annotations: {@link SqlRetry} and {@link Transactional}. The annotated methods
+ * will be called accordingly: 
+ * <ul>
+ *   <li>SQL errors in methods annotated with {@link SqlRetry} will be caught and the method will be re-executed</li>
+ *   <li>Methods annotated with {@link Transactional} will be executed after creating a transaction, and all operations done
+ *   via {@link MultiDataSourceJdbcResource}, {@link org.apache.hadoop.hive.metastore.txn.jdbc.ParameterizedQuery},
+ *   {@link org.apache.hadoop.hive.metastore.txn.jdbc.ParameterizedCommand} and 
+ *   {@link org.apache.hadoop.hive.metastore.txn.jdbc.QueryHandler} will use the created transaction.</li>
+ *   <li>In case a method is annotated with both annotations, the transaction will be inside the retry-call. This means 
+ *   in case of SQL errors and retries, the transaction will be rolled back and a new one will be created for each retry
+ *   attempt.</li>
+ * </ul> 
+ * Not annotated methods are called directly.
+ */
+public class ProxyTxnHandler implements InvocationHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(ProxyTxnHandler.class);
+
+  /**
+   * Gets the proxy interface for the given {@link TxnStore}.
+   * @param realStore The real {@link TxnStore} to proxy.
+   * @param sqlRetryHandler Responsible to re-execute the methods in case of failure.
+   * @return Returns the proxy object capable of retrying the failed calls automatically and transparently. 
+   */
+  public static TxnStore getProxy(TxnStore realStore, SqlRetryHandler sqlRetryHandler, MultiDataSourceJdbcResource jdbcResourceHandler) {
+    ProxyTxnHandler handler = new ProxyTxnHandler(realStore, sqlRetryHandler, jdbcResourceHandler);
+    return (TxnStore) Proxy.newProxyInstance(
+        ProxyTxnHandler.class.getClassLoader(),
+        new Class[]{ TxnStore.class },
+        handler);
+  }
+
+  private final SqlRetryHandler sqlRetryHandler;
+  private final TxnStore realStore;
+  private final MultiDataSourceJdbcResource jdbcResource;
+
+  private ProxyTxnHandler(TxnStore realStore, SqlRetryHandler sqlRetryHandler, MultiDataSourceJdbcResource jdbcResource) {
+    this.realStore = realStore;
+    this.sqlRetryHandler = sqlRetryHandler;
+    this.jdbcResource = jdbcResource;
+  }
+
+  @Override
+  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    String strArgs;
+    try {
+      strArgs = args == null ? "" : StringUtils.join(Arrays.asList(args), ",");
+    } catch (Exception e) {
+      LOG.warn("Error while trying to stringify the method arguments.", e);
+      strArgs = "unknown";
+    }
+    String callerId = method.getName() + "(" + strArgs + ")";
+    SqlRetry retry = method.getAnnotation(SqlRetry.class);
+    Transactional transactional = method.getAnnotation(Transactional.class);
+
+    ThrowingSupplier functionToCall = () -> {
+      try {
+        return method.invoke(realStore, args);
+      } catch (InvocationTargetException | UndeclaredThrowableException e) {
+        throw e.getCause();
+      }      
+    };
+    
+    if (transactional != null) {
+      ThrowingSupplier toCall = functionToCall;
+      functionToCall = () -> {        
+        LOG.debug("Invoking method within transactional context: {}", callerId);
+        TransactionContext context = null;
+        try {
+          jdbcResource.bindDataSource(transactional);
+          context = jdbcResource.getTransactionManager().getTransaction(transactional.propagation().value());
+          Object result = toCall.execute();
+          LOG.debug("Successfull method invocation within transactional context: {}, going to commit.", callerId);
+          jdbcResource.getTransactionManager().commit(context);
+          return result;
+        } catch (Exception e) {
+          if (Arrays.stream(transactional.noRollbackFor()).anyMatch(ex -> ex.isInstance(e)) ||
+              Arrays.stream(transactional.noRollbackForClassName()).anyMatch(exName -> exName.equals(e.getClass().getName()))) {
+            throw e;
+          }
+          if (context != null) {
+            if(transactional.rollbackFor().length > 0 || transactional.rollbackForClassName().length > 0) {

Review Comment:
   nit: space



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337126466


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java:
##########
@@ -461,9 +480,13 @@ void onRename(String oldCatName, String oldDbName, String oldTabName, String old
    * @return list of CompactionInfo structs.  These will not have id, type,
    * or runAs set since these are only potential compactions not actual ones.
    */
+  @SqlRetry
+  @Transactional(POOL_COMPACTOR)

Review Comment:
   should we set `@Transactional(POOL_COMPACTOR)` in CompactionTxnHandler?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337011321


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/NextCompactionHandler.java:
##########
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.FindNextCompactRequest;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.txn.jdbc.MultiDataSourceJdbcResource;
+import org.apache.hadoop.hive.metastore.txn.jdbc.QueryHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+
+import static org.apache.hadoop.hive.metastore.txn.TxnStore.INITIATED_STATE;
+import static org.apache.hadoop.hive.metastore.txn.TxnStore.WORKING_STATE;
+import static org.apache.hadoop.hive.metastore.txn.TxnUtils.getEpochFn;
+
+public class NextCompactionHandler implements QueryHandler<CompactionInfo> {
+
+  private static final Logger LOG = LoggerFactory.getLogger(NextCompactionHandler.class);
+
+  private final FindNextCompactRequest request;
+  private final MultiDataSourceJdbcResource retriableDatasourceTxnManager;
+  private final Timestamp currentDbTime;
+  private final long poolTimeout;
+
+  private static final String updateStatement =
+      "UPDATE \"COMPACTION_QUEUE\" " +
+          "SET " +
+          " \"CQ_WORKER_ID\" = :workerId, " +
+          " \"CQ_WORKER_VERSION\" = :workerVersion, " +
+          " \"CQ_START\" = :now, " +
+          " \"CQ_STATE\" = :newState " +
+          "WHERE \"CQ_ID\" = :id AND \"CQ_STATE\"= :oldState";
+
+  public NextCompactionHandler(FindNextCompactRequest request, MultiDataSourceJdbcResource retriableDatasourceTxnManager, Timestamp currentDbTime, long poolTimeout) {
+    this.request = request;
+    this.retriableDatasourceTxnManager = retriableDatasourceTxnManager;
+    this.currentDbTime = currentDbTime;
+    this.poolTimeout = poolTimeout;
+  }
+
+  @Override
+  public String getParameterizedQueryString(DatabaseProduct databaseProduct) throws MetaException {
+    StringBuilder sb = new StringBuilder();
+    sb.append("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
+        "\"CQ_TYPE\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\", " +
+        "\"CQ_TBLPROPERTIES\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_STATE\" = :state AND ");
+    boolean hasPoolName = StringUtils.isNotBlank(request.getPoolName());
+    if(hasPoolName) {
+      sb.append("\"CQ_POOL_NAME\"= :poolName");
+    } else {
+      sb.append("\"CQ_POOL_NAME\" IS NULL OR  \"CQ_ENQUEUE_TIME\" < (")
+          .append(getEpochFn(databaseProduct)).append(" - ").append(poolTimeout).append(")");
+    }
+    sb.append(" ORDER BY \"CQ_ID\" ASC");
+    return sb.toString();
+  }
+
+  @Override
+  public SqlParameterSource getQueryParameters() {
+    MapSqlParameterSource params = new MapSqlParameterSource()
+        .addValue("state", Character.toString(INITIATED_STATE), Types.CHAR);
+    if(StringUtils.isNotBlank(request.getPoolName())) {
+      params.addValue("poolName", request.getPoolName());
+    }
+    return params;
+  }
+
+  @Override
+  public CompactionInfo extractData(ResultSet rs) throws SQLException, DataAccessException {
+    CompactionInfo info;
+    while (rs.next()){

Review Comment:
   should be if



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1721242280

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [83 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309797350


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/DataSourceWrapper.java:
##########
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+import org.springframework.jdbc.datasource.DataSourceTransactionManager;
+import org.springframework.jdbc.datasource.DataSourceUtils;
+import org.springframework.transaction.PlatformTransactionManager;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.TransactionException;
+import org.springframework.transaction.TransactionStatus;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Wraps multiple {@link DataSource}s into a single object and offers transaction management functionality. 
+ * Allows access of the {@link NamedParameterJdbcTemplate}, {@link Connection} objects associated with the wrapped datasources.
+ */
+public class DataSourceWrapper {
+
+  private static final Logger LOG = LoggerFactory.getLogger(DataSourceWrapper.class);
+  private static final ThreadLocal<RetryContext> threadLocal = new ThreadLocal<>();
+
+  private final Map<String, DataSource> dataSources;
+  private final Map<String, PlatformTransactionManager> transactionManagers = new HashMap<>();
+  private final Map<String, NamedParameterJdbcTemplate> jdbcTemplates = new HashMap<>();
+  private final DatabaseProduct databaseProduct;
+
+  /**
+   * Creates a new isntance of the {@link DataSourceWrapper} class
+   * @param dataSources A {@link Map} of the datasource names and the corresponding datasources which needs to be wrapped
+   *                    by this instance
+   * @param databaseProduct A {@link DatabaseProduct} instance representing the type of the underlying HMS dabatabe.
+   */
+  public DataSourceWrapper(Map<String, DataSource> dataSources, DatabaseProduct databaseProduct) {
+    this.dataSources = dataSources;
+    for(String dataSource : dataSources.keySet()) {
+      NamedParameterJdbcTemplate jdbcTemplate = new NamedParameterJdbcTemplate(dataSources.get(dataSource));
+      jdbcTemplates.put(dataSource, jdbcTemplate);
+      transactionManagers.put(dataSource, new DataSourceTransactionManager(Objects.requireNonNull(jdbcTemplate.getJdbcTemplate().getDataSource())));

Review Comment:
   why such a complex construct, don't we have a datasource in `dataSources.get(dataSource)`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310119152


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6246,18 +6303,24 @@ public void releaseLocks(Long timestamp) {
         LOG.warn("Unable to update MT_KEY2 value for MT_KEY1=" + key, ex);
         rollbackDBConn(dbConn);
       }
-      close(rs, stmt, dbConn);
+      TxnHandler.close(rs, stmt, dbConn);
       if(derbySemaphore != null) {
         derbySemaphore.release();
       }
       if (LOG.isDebugEnabled()) {
         LOG.debug("{} unlocked by {}", quoteString(key), quoteString(TxnHandler.hostname));
       }
     }
+
+    @Override
+    public void close() {
+      releaseLocks();
+    }
+
   }
 
 
-  private static class NoPoolConnectionPool implements DataSource {
+  public static class NoPoolConnectionPool implements DataSource {

Review Comment:
   fixed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308820192


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -289,109 +125,12 @@ public CompactionInfo findNextToCompact(String workerId) throws MetaException {
    */
   @Override
   @RetrySemantics.SafeToRetry
-  @SuppressWarnings("squid:S2095")
   public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws MetaException {
-    try {
-      if (rqst == null) {
-        throw new MetaException("FindNextCompactRequest is null");
-      }
-
-      Connection dbConn = null;
-      PreparedStatement stmt = null;
-      //need a separate stmt for executeUpdate() otherwise it will close the ResultSet(HIVE-12725)
-      Statement updStmt = null;
-      ResultSet rs = null;
-
-      long poolTimeout = MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_WORKER_POOL_TIMEOUT, TimeUnit.MILLISECONDS);
-
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        StringBuilder sb = new StringBuilder();
-        sb.append("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
-          "\"CQ_TYPE\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\", " +
-          "\"CQ_TBLPROPERTIES\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_STATE\" = " + quoteChar(INITIATED_STATE) + " AND ");
-        boolean hasPoolName = StringUtils.isNotBlank(rqst.getPoolName());
-        if(hasPoolName) {
-          sb.append("\"CQ_POOL_NAME\"=?");
-        } else {
-          sb.append("\"CQ_POOL_NAME\" IS NULL OR  \"CQ_ENQUEUE_TIME\" < (")
-            .append(getEpochFn(dbProduct)).append(" - ").append(poolTimeout).append(")");
-        }
-        sb.append(" ORDER BY \"CQ_ID\" ASC");
-        String query = sb.toString();
-        stmt = dbConn.prepareStatement(query);
-        if (hasPoolName) {
-          stmt.setString(1, rqst.getPoolName());
-        }
-
-        LOG.debug("Going to execute query <{}>", query);
-        rs = stmt.executeQuery();
-        if (!rs.next()) {
-          LOG.debug("No compactions found ready to compact");
-          dbConn.rollback();
-          return null;
-        }
-        updStmt = dbConn.createStatement();
-        do {
-          CompactionInfo info = new CompactionInfo();
-          info.id = rs.getLong(1);
-          info.dbname = rs.getString(2);
-          info.tableName = rs.getString(3);
-          info.partName = rs.getString(4);
-          info.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(5).charAt(0));
-          info.poolName = rs.getString(6);
-          info.numberOfBuckets = rs.getInt(7);
-          info.orderByClause = rs.getString(8);
-          info.properties = rs.getString(9);
-          info.workerId = rqst.getWorkerId();
-
-          String workerId = rqst.getWorkerId();
-          String workerVersion = rqst.getWorkerVersion();
-          String workerIdSqlValue = (workerId == null) ? "NULL" : ("'" + workerId + "'");
-
-          // Now, update this record as being worked on by this worker.
-          long now = getDbTime(dbConn);
-          query = "" +
-              "UPDATE " +
-              "  \"COMPACTION_QUEUE\" " +
-              "SET " +
-              " \"CQ_WORKER_ID\" = " + workerIdSqlValue + ", " +
-              " \"CQ_WORKER_VERSION\" = '" + workerVersion + "', " +
-              " \"CQ_START\" = " + now + ", " +
-              " \"CQ_STATE\" = '" + WORKING_STATE + "' " +
-              "WHERE \"CQ_ID\" = " + info.id +
-              "  AND \"CQ_STATE\"='" + INITIATED_STATE + "'";
-
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = updStmt.executeUpdate(query);
-          if(updCount == 1) {
-            dbConn.commit();
-            return info;
-          }
-          if(updCount == 0) {
-            LOG.debug("Worker {} (version: {}) picked up {}", workerId, workerVersion, info);
-            continue;
-          }
-          LOG.error("Unable to set to cq_state={} for compaction record: {}. updCnt={}. workerId={}. workerVersion={}",
-              WORKING_STATE, info, updCount, workerId, workerVersion);
-          dbConn.rollback();
-          return null;
-        } while( rs.next());
-        dbConn.rollback();
-        return null;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for compaction, " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "findNextToCompact(rqst:" + rqst + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(updStmt);
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return findNextToCompact(rqst);
+    if (rqst == null) {
+      throw new MetaException("FindNextCompactRequest is null");
     }
+    long poolTimeout = MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_WORKER_POOL_TIMEOUT, TimeUnit.MILLISECONDS);

Review Comment:
   is this a generic compaction pool timeout or just this query specific?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308996345


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,38 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).execute(jdbcTemplate);
   }
-
+  
   /**
    * Clean up entries from TXN_TO_WRITE_ID table less than min_uncommited_txnid as found by
    * min(max(TXNS.txn_id), min(WRITE_SET.WS_COMMIT_ID), min(Aborted TXNS.txn_id)).
    */
   @Override
   @RetrySemantics.SafeToRetry
   public void cleanTxnToWriteIdTable() throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-
-      try {
-        long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
-
-        // We query for minimum values in all the queries and they can only increase by any concurrent
-        // operations. So, READ COMMITTED is sufficient.
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-
-        // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
-        // If there are no txns which are currently open or aborted in the system, then current value of
-        // max(TXNS.txn_id) could be min_uncommitted_txnid.
-        String s = "SELECT MIN(\"RES\".\"ID\") AS \"ID\" FROM (" +
-          " SELECT MAX(\"TXN_ID\") + 1 AS \"ID\" FROM \"TXNS\"" +
-          (useMinHistoryLevel ? "" :
-          "   UNION" +
-          " SELECT MIN(\"WS_TXNID\") AS \"ID\" FROM \"WRITE_SET\"") +
-          "   UNION" +
-          " SELECT MIN(\"TXN_ID\") AS \"ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          (useMinHistoryLevel ? "" :
-          "   OR \"TXN_STATE\" = " + TxnStatus.OPEN) +
-          " ) \"RES\"";
+    long minTxnIdSeenOpen = findMinTxnIdSeenOpen();
 
-        LOG.debug("Going to execute query <{}>", s);
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly initialized, no record found in TXNS");
-        }
-        long minUncommitedTxnid = Math.min(rs.getLong(1), minTxnIdSeenOpen);
-
-        // As all txns below min_uncommitted_txnid are either committed or empty_aborted, we are allowed
-        // to cleanup the entries less than min_uncommitted_txnid from the TXN_TO_WRITE_ID table.
-        s = "DELETE FROM \"TXN_TO_WRITE_ID\" WHERE \"T2W_TXNID\" < " + minUncommitedTxnid;
-        LOG.debug("Going to execute delete <{}>", s);
-        int rc = stmt.executeUpdate(s);
-        LOG.info("Removed {} rows from TXN_TO_WRITE_ID with Txn Low-Water-Mark: {}", rc, minUncommitedTxnid);
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from TXN_TO_WRITE_ID table " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "cleanTxnToWriteIdTable");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      cleanTxnToWriteIdTable();
+    // First need to find the min_uncommitted_txnid which is currently seen by any open transactions.
+    // If there are no txns which are currently open or aborted in the system, then current value of
+    // max(TXNS.txn_id) could be min_uncommitted_txnid.
+    Long minTxnId = jdbcTemplate.execute(new MinUncommittedTxnIdHandler(useMinHistoryLevel));
+    if (minTxnId == null) {

Review Comment:
   why that is not part of MinUncommittedTxnIdHandler?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308982378


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -615,57 +190,19 @@ public void markCleanerStart(CompactionInfo info) throws MetaException {
   @RetrySemantics.ReadOnly
   public void clearCleanerStart(CompactionInfo info) throws MetaException {
     LOG.debug("Running clearCleanerStart with CompactionInfo: {}", info);
-
-    try {
-      Connection dbConn = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        setCleanerStart(dbConn, info, -1L);
-      } catch (SQLException e) {
-        LOG.error("Unable to clear the cleaner start time for compaction record  " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "clearCleanerStart(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      clearCleanerStart(info);
-    }
+    setCleanerStart(info, -1L);
   }
 
-  private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timestamp)
-      throws RetryException, SQLException {
-    long id = info.id;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      String query = "" +
-          " UPDATE " +
-          "  \"COMPACTION_QUEUE\" " +
-          " SET " +
-          "  \"CQ_CLEANER_START\" = " + timestamp +
-          " WHERE " +
-          "  \"CQ_ID\" = " + id +
-          " AND " +
-          "  \"CQ_STATE\"='" + READY_FOR_CLEANING + "'";
-
-      pStmt = dbConn.prepareStatement(query);
-      LOG.debug("Going to execute update <{}> for CQ_ID={}", query, id);
-      int updCount = pStmt.executeUpdate();
-      if (updCount != 1) {
-        LOG.error("Unable to update compaction record: {}.  Update count={}", info, updCount);
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-      } else {
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      }
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+  private void setCleanerStart(CompactionInfo info, Long timestamp) throws MetaException {
+    jdbcTemplate.execute(
+        " UPDATE \"COMPACTION_QUEUE\" " +

Review Comment:
   wouldn't that create multiple string objects? should we move static queries in constants?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310037354


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -6246,18 +6303,24 @@ public void releaseLocks(Long timestamp) {
         LOG.warn("Unable to update MT_KEY2 value for MT_KEY1=" + key, ex);
         rollbackDBConn(dbConn);
       }
-      close(rs, stmt, dbConn);
+      TxnHandler.close(rs, stmt, dbConn);
       if(derbySemaphore != null) {
         derbySemaphore.release();
       }
       if (LOG.isDebugEnabled()) {
         LOG.debug("{} unlocked by {}", quoteString(key), quoteString(TxnHandler.hostname));
       }
     }
+
+    @Override
+    public void close() {
+      releaseLocks();
+    }
+
   }
 
 
-  private static class NoPoolConnectionPool implements DataSource {
+  public static class NoPoolConnectionPool implements DataSource {

Review Comment:
   does it have to be public?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310070413


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1603,21 +1645,24 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
         }
 
         if (!isHiveReplTxn) {
-          createCommitNotificationEvent(dbConn, txnid , txnType);
+          createCommitNotificationEvent(jdbcTemplate, txnid , txnType);
         }
 
         LOG.debug("Going to commit");
-        dbConn.commit();
+        jdbcTemplate.commit();
 
         if (MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.METASTORE_ACIDMETRICS_EXT_ON)) {
           Metrics.getOrCreateCounter(MetricsConstants.TOTAL_NUM_COMMITTED_TXNS).inc();
         }
       } catch (SQLException e) {
         LOG.debug("Going to rollback: ", e);
-        rollbackDBConn(dbConn);
+        jdbcTemplate.rollback();

Review Comment:
   i don't get it, we did refactor in some methods, replaced retry with retryhandler and used jdbctemplatewrapper. here we just using jdbcTemplate to access the connection object?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1308773354


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -58,114 +68,18 @@
  * methods are not available through the thrift interface.
  */
 class CompactionTxnHandler extends TxnHandler {
-  private static final String CLASS_NAME = CompactionTxnHandler.class.getName();
-  private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
-  private static final String DB_FAILED_TO_CONNECT = "Unable to connect to transaction database: ";
-
-  private static DataSource connPoolCompaction;
-
-  private static final String SELECT_COMPACTION_QUEUE_BY_TXN_ID =
-      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-          + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-          + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-          + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-          + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-          + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" "
-          + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?";
-  private static final String SELECT_COMPACTION_METRICS_CACHE_QUERY =
-      "SELECT \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM \"COMPACTION_METRICS_CACHE\" " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ?";
-  private static final String NO_SELECT_COMPACTION_METRICS_CACHE_FOR_TYPE_QUERY =
-      "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM " +
-      "\"COMPACTION_METRICS_CACHE\" WHERE \"CMC_METRIC_TYPE\" = ? ORDER BY \"CMC_METRIC_VALUE\" DESC";
-  private static final String UPDATE_COMPACTION_METRICS_CACHE_QUERY =
-      "UPDATE \"COMPACTION_METRICS_CACHE\" SET \"CMC_METRIC_VALUE\" = ?, \"CMC_VERSION\" = ? " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ? " +
-          "AND \"CMC_VERSION\" = ?";
-  private static final String INSERT_COMPACTION_METRICS_CACHE_QUERY =
-      "INSERT INTO \"COMPACTION_METRICS_CACHE\" ( " +
-          "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_TYPE\", \"CMC_METRIC_VALUE\", " +
-          "\"CMC_VERSION\" ) VALUES (?, ?, ?, ?, ?, ?)";
-  private static final String DELETE_COMPACTION_METRICS_CACHE_QUERY =
-      "DELETE FROM \"COMPACTION_METRICS_CACHE\" WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? " +
-          "AND \"CMC_METRIC_TYPE\" = ?";
-
-  private static final String DELETE_FAILED_TXNS_SQL =
-      "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
-          "AND (\"TXN_STATE\" = " + TxnStatus.ABORTED + " OR \"TXN_STATE\" = " + TxnStatus.COMMITTED + ") " +
-          "AND \"TXN_ID\" < ?";
-
-  // Three inner sub-queries which are under left-join to fetch the required data for aborted txns.
-  private static final String SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY =
-          " \"res1\".\"TC_DATABASE\" AS \"DB\", \"res1\".\"TC_TABLE\" AS \"TBL\", \"res1\".\"TC_PARTITION\" AS \"PART\", " +
-          " \"res1\".\"MIN_TXN_START_TIME\" AS \"MIN_TXN_START_TIME\", \"res1\".\"ABORTED_TXN_COUNT\" AS \"ABORTED_TXN_COUNT\", " +
-          " \"res2\".\"MIN_OPEN_WRITE_TXNID\" AS \"MIN_OPEN_WRITE_TXNID\", \"res3\".\"RETRY_RETENTION\" AS \"RETRY_RETENTION\", " +
-          " \"res3\".\"ID\" AS \"RETRY_CQ_ID\" " +
-      " FROM " +
-          // First sub-query - Gets the aborted txns with min txn start time, number of aborted txns
-          // for corresponding db, table, partition.
-          " ( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TXN_STARTED\") AS \"MIN_TXN_START_TIME\", " +
-          " COUNT(*) AS \"ABORTED_TXN_COUNT\" FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" %s ) \"res1\" " +
-      " LEFT JOIN" +
-          // Second sub-query - Gets the min open txn id for corresponding db, table, partition.
-          "( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TC_TXNID\") AS \"MIN_OPEN_WRITE_TXNID\" " +
-          " FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" ) \"res2\"" +
-      " ON \"res1\".\"TC_DATABASE\" = \"res2\".\"TC_DATABASE\"" +
-      " AND \"res1\".\"TC_TABLE\" = \"res2\".\"TC_TABLE\"" +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res2\".\"TC_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res2\".\"TC_PARTITION\" IS NULL)) " +
-      " LEFT JOIN " +
-          // Third sub-query - Gets the retry entries for corresponding db, table, partition.
-          "( SELECT \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", MAX(\"CQ_ID\") AS \"ID\", " +
-          " MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RETENTION\", " +
-          " MIN(\"CQ_COMMIT_TIME\") - %s + MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RECORD_CHECK\" FROM \"COMPACTION_QUEUE\" " +
-          " WHERE \"CQ_TYPE\" = " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"res3\" " +
-      " ON \"res1\".\"TC_DATABASE\" = \"res3\".\"CQ_DATABASE\" " +
-      " AND \"res1\".\"TC_TABLE\" = \"res3\".\"CQ_TABLE\" " +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res3\".\"CQ_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res3\".\"CQ_PARTITION\" IS NULL))" +
-      " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL";
-
-  private static final String DELETE_CQ_ENTRIES = "DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-
-  private static final String DELETE_CQ_AND_ABORT_ENTRIES = DELETE_CQ_ENTRIES +
-      " OR ( \"CQ_DATABASE\" = ? AND \"CQ_TABLE\" = ? " +
-      " AND \"CQ_PARTITION\" %s AND \"CQ_TYPE\" = " +
-      quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) + " )";
-
-  private static final String INSERT_ABORT_RETRY_ENTRY_INTO_COMPACTION_QUEUE =
-      "INSERT INTO \"COMPACTION_QUEUE\" (\"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
-      " \"CQ_TYPE\", \"CQ_STATE\", \"CQ_RETRY_RETENTION\", \"CQ_ERROR_MESSAGE\", \"CQ_COMMIT_TIME\") " +
-      " VALUES (?, ?, ?, ?, ?, ?, ?, ?, %s)";
+  
+  private static final Logger LOG = LoggerFactory.getLogger(CompactionTxnHandler.class.getName());
 
   public CompactionTxnHandler() {
   }
 
-  @Override
-  public void setConf(Configuration conf) {
-    super.setConf(conf);
-    synchronized (CompactionTxnHandler.class) {
-      if (connPoolCompaction == null) {
-        int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);

Review Comment:
   that part. should stay here, see another comment with more details



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309004327


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -678,316 +215,38 @@ private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timest
   @RetrySemantics.CannotRetry
   public void markCleaned(CompactionInfo info) throws MetaException {
     LOG.debug("Running markCleaned with CompactionInfo: {}", info);
-    try {
-      Connection dbConn = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        if (!info.isAbortedTxnCleanup()) {
-          String s = "INSERT INTO \"COMPLETED_COMPACTIONS\"(\"CC_ID\", \"CC_DATABASE\", "
-              + "\"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", "
-              + "\"CC_START\", \"CC_END\", \"CC_RUN_AS\", \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", "
-              + "\"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", \"CC_ENQUEUE_TIME\", "
-              + "\"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\", "
-              + "\"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\","
-              + "\"CC_ORDER_BY\") "
-              + "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-              + quoteChar(SUCCEEDED_STATE) + ", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", "
-              + getEpochFn(dbProduct) + ", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", "
-              + "\"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", \"CQ_ENQUEUE_TIME\", "
-              + "\"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-              + "\"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", \"CQ_NUMBER_OF_BUCKETS\", "
-              + "\"CQ_ORDER_BY\" "
-              + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-          pStmt = dbConn.prepareStatement(s);
-          pStmt.setLong(1, info.id);
-          LOG.debug("Going to execute update <{}> for CQ_ID={}", s, info.id);
-          pStmt.executeUpdate();
-        }
-
-        /* Remove compaction queue record corresponding to the compaction which has been successful as well as
-         * remove all abort retry associated metadata of table/partition in the COMPACTION_QUEUE both when compaction
-         * or abort cleanup is successful. We don't want a situation wherein we have an abort retry entry for a table
-         * but no corresponding entry in TXN_COMPONENTS table. Successful compaction will delete
-         * the retry metadata, so that abort cleanup is retried again (an optimistic retry approach).
-         */
-        removeCompactionAndAbortRetryEntries(dbConn, info);
-
-        if (!info.isAbortedTxnCleanup()) {
-          // Remove entries from completed_txn_components as well, so we don't start looking there
-          // again but only up to the highest write ID include in this compaction job.
-          //highestWriteId will be NULL in upgrade scenarios
-          String query =
-              "DELETE FROM \"COMPLETED_TXN_COMPONENTS\" " +
-              " WHERE \"CTC_DATABASE\" = ? AND \"CTC_TABLE\" = ?";
-          if (info.partName != null) {
-            query += " AND \"CTC_PARTITION\" = ?";
-          }
-          if (info.highestWriteId != 0) {
-            query += " AND \"CTC_WRITEID\" <= ?";
-          }
-          pStmt = dbConn.prepareStatement(query);
-          int paramCount = 1;
-          pStmt.setString(paramCount++, info.dbname);
-          pStmt.setString(paramCount++, info.tableName);
-          if (info.partName != null) {
-            pStmt.setString(paramCount++, info.partName);
-          }
-          if (info.highestWriteId != 0) {
-            pStmt.setLong(paramCount, info.highestWriteId);
-          }
-          LOG.debug("Going to execute update <{}>", query);
-          int updCount = pStmt.executeUpdate();
-          if (updCount < 1) {
-            LOG.warn("Expected to remove at least one row from completed_txn_components when " +
-                    "marking compaction entry as clean!");
-          }
-          LOG.debug("Removed {} records from completed_txn_components", updCount);
-        }
-
-        // Do cleanup of metadata in TXN_COMPONENTS table.
-        removeTxnComponents(dbConn, info);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCleaned(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  private void removeTxnComponents(Connection dbConn, CompactionInfo info) throws MetaException, RetryException {
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      /*
-       * compaction may remove data from aborted txns above tc_writeid bit it only guarantees to
-       * remove it up to (inclusive) tc_writeid, so it's critical to not remove metadata about
-       * aborted TXN_COMPONENTS above tc_writeid (and consequently about aborted txns).
-       * See {@link ql.txn.compactor.Cleaner.removeFiles()}
-       */
-      String s = "DELETE FROM \"TXN_COMPONENTS\" WHERE \"TC_TXNID\" IN ( "
-              + "SELECT \"TXN_ID\" FROM \"TXNS\" WHERE \"TXN_STATE\" = " + TxnStatus.ABORTED + ") "
-              + "AND \"TC_DATABASE\" = ? AND \"TC_TABLE\" = ? "
-              + "AND \"TC_PARTITION\" " + TxnUtils.nvl(info.partName);
-
-      List<String> queries = new ArrayList<>();
-      Iterator<Long> writeIdsIter = null;
-      List<Integer> counts = null;
-
-      if (info.writeIds != null && !info.writeIds.isEmpty()) {
-        StringBuilder prefix = new StringBuilder(s).append(" AND ");
-        List<String> questions = Collections.nCopies(info.writeIds.size(), "?");
-
-        counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix,
-                new StringBuilder(), questions, "\"TC_WRITEID\"", false, false);
-        writeIdsIter = info.writeIds.iterator();
-      } else if (!info.hasUncompactedAborts) {
-        if (info.highestWriteId != 0) {
-          s += " AND \"TC_WRITEID\" <= ?";
-        }
-        queries.add(s);
-      }
-
-      for (int i = 0; i < queries.size(); i++) {
-        String query = queries.get(i);
-        int writeIdCount = (counts != null) ? counts.get(i) : 0;
-
-        LOG.debug("Going to execute update <{}>", query);
-        pStmt = dbConn.prepareStatement(query);
-        int paramCount = 1;
-
-        pStmt.setString(paramCount++, info.dbname);
-        pStmt.setString(paramCount++, info.tableName);
-        if (info.partName != null) {
-          pStmt.setString(paramCount++, info.partName);
-        }
-        if (info.highestWriteId != 0 && writeIdCount == 0) {
-          pStmt.setLong(paramCount, info.highestWriteId);
-        }
-        for (int j = 0; j < writeIdCount; j++) {
-          if (writeIdsIter.hasNext()) {
-            pStmt.setLong(paramCount + j, writeIdsIter.next());
-          }
-        }
-        int rc = pStmt.executeUpdate();
-        LOG.debug("Removed {} records from txn_components", rc);
-      }
-    } catch (SQLException e) {
-      LOG.error("Unable to delete from txn components due to {}", e.getMessage());
-      LOG.debug("Going to rollback");
-      rollbackDBConn(dbConn);
-      checkRetryable(e, "markCleanedForAborts(" + info + ")");
-      throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+    new MarkCleanedFunction(info, dbProduct, conf).execute(jdbcTemplate);

Review Comment:
   why do you need to pass the product, doesn't the wrapper already have it?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309883605


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2557,7 +2583,7 @@ public void performWriteSetGC() throws MetaException {
     }
   }
 
-  protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException, SQLException {
+  protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException {

Review Comment:
   getMinOpenTxnIdWaterMark() is called from multiple places and `TxnHandler` cleanup is not part of this task. I made only the really necessary changes in it. Since place where this method is called in `TxnHandler` are still not using `jdbcTemplate`, therefore this method must still accept a `Connection` instance as parameter. With the cleanup of `TxnHandler` also, the signature can finally be changed and `jdbcTemplate` used instead.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293066381


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1396,245 +388,59 @@ private void insertAbortRetryRetentionTimeOnError(Connection dbConn, CompactionI
   @Override
   @RetrySemantics.SafeToRetry
   public void purgeCompactionHistory() throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    List<Long> deleteSet = new ArrayList<>();
-    RetentionCounters rc = null;
-    long timeoutThreshold = System.currentTimeMillis() -
-            MetastoreConf.getTimeVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_TIMEOUT, TimeUnit.MILLISECONDS);
-    int didNotInitiateRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_DID_NOT_INITIATE);
-    int failedRetention = getFailedCompactionRetention();
-    int succeededRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_SUCCEEDED);
-    int refusedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_REFUSED);
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        /* cc_id is monotonically increasing so for any entity sorts in order of compaction history,
-        thus this query groups by entity and withing group sorts most recent first */
-        rs = stmt.executeQuery("SELECT \"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", "
-            + "\"CC_STATE\" , \"CC_START\", \"CC_TYPE\" "
-            + "FROM \"COMPLETED_COMPACTIONS\" ORDER BY \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\"," +
-                "\"CC_ID\" DESC");
-        String lastCompactedEntity = null;
-        /* In each group, walk from most recent and count occurrences of each state type.  Once you
-        * have counted enough (for each state) to satisfy retention policy, delete all other
-        * instances of this status, plus timed-out entries (see this method's JavaDoc).
-        */
-        while(rs.next()) {
-          CompactionInfo ci = new CompactionInfo(
-              rs.getLong(1), rs.getString(2), rs.getString(3),
-              rs.getString(4), rs.getString(5).charAt(0));
-          ci.start = rs.getLong(6);
-          ci.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(7).charAt(0));
-          if(!ci.getFullPartitionName().equals(lastCompactedEntity)) {
-            lastCompactedEntity = ci.getFullPartitionName();
-            rc = new RetentionCounters(didNotInitiateRetention, failedRetention, succeededRetention, refusedRetention);
-          }
-          checkForDeletion(deleteSet, ci, rc, timeoutThreshold);
-        }
-        close(rs);
-
-        if (deleteSet.size() <= 0) {
-          return;
-        }
-
-        List<String> queries = new ArrayList<>();
-
-        StringBuilder prefix = new StringBuilder();
-        StringBuilder suffix = new StringBuilder();
-
-        prefix.append("DELETE FROM \"COMPLETED_COMPACTIONS\" WHERE ");
-
-        List<String> questions = new ArrayList<>(deleteSet.size());
-        for (int  i = 0; i < deleteSet.size(); i++) {
-          questions.add("?");
-        }
-        List<Integer> counts = TxnUtils.buildQueryWithINClauseStrings(conf, queries, prefix, suffix, questions,
-            "\"CC_ID\"", false, false);
-        int totalCount = 0;
-        for (int i = 0; i < queries.size(); i++) {
-          String query = queries.get(i);
-          long insertCount = counts.get(i);
-          LOG.debug("Going to execute update <{}>", query);
-          pStmt = dbConn.prepareStatement(query);
-          for (int j = 0; j < insertCount; j++) {
-            pStmt.setLong(j + 1, deleteSet.get(totalCount + j));
-          }
-          totalCount += insertCount;
-          int count = pStmt.executeUpdate();
-          LOG.debug("Removed {} records from COMPLETED_COMPACTIONS", count);
-        }
-        dbConn.commit();
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "purgeCompactionHistory()");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        close(rs, stmt, dbConn);
-        closeStmt(pStmt);
-      }
-    } catch (RetryException ex) {
-      purgeCompactionHistory();
-    }
-  }
-
-  /**
-   * this ensures that the number of failed compaction entries retained is > than number of failed
-   * compaction threshold which prevents new compactions from being scheduled.
-   */
-  private int getFailedCompactionRetention() {
-    int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-    int failedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED);
-    if(failedRetention < failedThreshold) {
-      LOG.warn("Invalid configuration {}={} < {}={}.  Will use {}={}",
-          ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.getVarname(), failedRetention,
-          ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED, failedRetention,
-          ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.getVarname(), failedRetention);
-      failedRetention = failedThreshold;
-    }
-    return failedRetention;
+    new PurgeCompactionHistoryFunction(conf).call(dataSourceWrapper);
   }
 
   /**
    * Returns {@code true} if there already exists sufficient number of consecutive failures for
    * this table/partition so that no new automatic compactions will be scheduled.
    * User initiated compactions don't do this check.
-   *
    * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
    * That would be a meta operations, i.e. first find all partitions for this table (which have
    * txn info) and schedule each compaction separately.  This avoids complications in this logic.
    */
   @Override
   @RetrySemantics.ReadOnly
   public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
-    Connection dbConn = null;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        pStmt = dbConn.prepareStatement("SELECT \"CC_STATE\", \"CC_ENQUEUE_TIME\" FROM \"COMPLETED_COMPACTIONS\" WHERE " +
-          "\"CC_DATABASE\" = ? AND " +
-          "\"CC_TABLE\" = ? " +
-          (ci.partName != null ? "AND \"CC_PARTITION\" = ?" : "") +
-          " AND \"CC_STATE\" != " + quoteChar(DID_NOT_INITIATE) + " ORDER BY \"CC_ID\" DESC");
-        pStmt.setString(1, ci.dbname);
-        pStmt.setString(2, ci.tableName);
-        if (ci.partName != null) {
-          pStmt.setString(3, ci.partName);
-        }
-        rs = pStmt.executeQuery();
-        int numFailed = 0;
-        int numTotal = 0;
-        long lastEnqueueTime = -1;
-        int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-        while(rs.next() && ++numTotal <= failedThreshold) {
-          long enqueueTime = rs.getLong(2);
-          if (enqueueTime > lastEnqueueTime) {
-            lastEnqueueTime = enqueueTime;
-          }
-          if(rs.getString(1).charAt(0) == FAILED_STATE) {
-            numFailed++;
-          }
-          else {
-            numFailed--;
-          }
-        }
-        // If the last attempt was too long ago, ignore the failed threshold and try compaction again
-        long retryTime = MetastoreConf.getTimeVar(conf,
-            ConfVars.COMPACTOR_INITIATOR_FAILED_RETRY_TIME, TimeUnit.MILLISECONDS);
-        boolean needsRetry = (retryTime > 0) && (lastEnqueueTime + retryTime < System.currentTimeMillis());
-        return (numFailed == failedThreshold) && !needsRetry;
-      }
-      catch (SQLException e) {
-        LOG.error("Unable to check for failed compactions", e);
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "checkFailedCompactions(" + ci + ")");
-        LOG.error(DB_FAILED_TO_CONNECT, e);
-        return false;//weren't able to check
-      } finally {
-        close(rs, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return checkFailedCompactions(ci);
-    }
+    return new CheckFailedCompactionsHandler(conf, ci).execute(dataSourceWrapper);
   }
 
-
   private void updateStatus(CompactionInfo ci) throws MetaException {
     String strState = CompactionState.fromSqlConst(ci.state).toString();
+
     LOG.debug("Marking as {}: CompactionInfo: {}", strState, ci);
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        pStmt = dbConn.prepareStatement("SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-                + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-                + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-                + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-                + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-                + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?");
-        pStmt.setLong(1, ci.id);
-        rs = pStmt.executeQuery();
-        if (rs.next()) {
-          //preserve errorMessage and state
-          String errorMessage = ci.errorMessage;
-          char state = ci.state;
-          ci = CompactionInfo.loadFullFromCompactionQueue(rs);
-          ci.errorMessage = errorMessage;
-          ci.state = state;
+    CompactionInfo ciActual = new GetCompactionInfoHandler(ci.id, false).execute(dataSourceWrapper); 
 
-          pStmt = dbConn.prepareStatement(DELETE_CQ_ENTRIES);
-          pStmt.setLong(1, ci.id);
-          LOG.debug("Going to execute update <{}>", DELETE_CQ_ENTRIES);
-          pStmt.executeUpdate();
-        }
-        else {
-          if(ci.id > 0) {
-            //the record with valid CQ_ID has disappeared - this is a sign of something wrong
-            throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
-          }
-        }
-        if(ci.id == 0) {
-          //The failure occurred before we even made an entry in COMPACTION_QUEUE
-          //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
-          ci.id = generateCompactionQueueId(stmt);
-          //this is not strictly accurate, but 'type' cannot be null.
-          if(ci.type == null) {
-            ci.type = CompactionType.MINOR;
-          }
-          ci.start = getDbTime(dbConn);
-          LOG.debug("The failure occurred before we even made an entry in COMPACTION_QUEUE. Generated ID so that we "
-                  + "can make an entry in COMPLETED_COMPACTIONS. New Id: {}", ci.id);
-        }
-        close(rs, stmt, null);
-        closeStmt(pStmt);
+    long endTime = getDbTime().getTime();
+    if (ciActual != null) {
+      //preserve errorMessage and state
+      ciActual.errorMessage = ci.errorMessage;
+      ciActual.state = ci.state;
 
-        pStmt = dbConn.prepareStatement(TxnQueries.INSERT_INTO_COMPLETED_COMPACTION);
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        LOG.debug("Inserted {} entries into COMPLETED_COMPACTIONS", updCount);
-        closeStmt(pStmt);
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Failed to mark compaction request as " + strState + ", rolling back transaction: " + ci, e);
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "updateStatus(" + ci + ")");
-      } finally {
-        close(rs, stmt, null);
-        close(null, pStmt, dbConn);
+      dataSourceWrapper.getJdbcTemplate().update("DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id",
+          new MapSqlParameterSource("id", ci.id));
+    } else {
+      if (ci.id > 0) {
+        //the record with valid CQ_ID has disappeared - this is a sign of something wrong
+        throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
+      }
+      ciActual = ci;
+    }
+    if (ciActual.id == 0) {
+      //The failure occurred before we even made an entry in COMPACTION_QUEUE
+      //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
+      ciActual.id = generateCompactionQueueId();
+      //this is not strictly accurate, but 'type' cannot be null.
+      if (ciActual.type == null) {
+        ciActual.type = CompactionType.MINOR;
       }
-    } catch (RetryException e) {
-      updateStatus(ci);
+      //in case of creating a new entry start and end time will be the same
+      ciActual.start = endTime;
+      LOG.debug("The failure occurred before we even made an entry in COMPACTION_QUEUE. Generated ID so that we "
+          + "can make an entry in COMPLETED_COMPACTIONS. New Id: {}", ciActual.id);
     }
+
+    new InsertCompactionInfoCommand(ciActual, endTime).execute(dataSourceWrapper);

Review Comment:
   not consistent naming, in some places it's handler, in other  - command. also not consistent approach, mix of jdbcTemplace and 1 liner command



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293071008


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -402,37 +141,12 @@ public CompactionInfo findNextToCompact(FindNextCompactRequest rqst) throws Meta
   @Override
   @RetrySemantics.SafeToRetry
   public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        stmt = dbConn.createStatement();
-        String s = "UPDATE \"COMPACTION_QUEUE\" SET \"CQ_STATE\" = '" + READY_FOR_CLEANING + "', "
-            + "\"CQ_WORKER_ID\" = NULL"
-            + " WHERE \"CQ_ID\" = " + info.id;
-        LOG.debug("Going to execute update <{}>", s);
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state={} for compaction record: {}. updCnt={}", READY_FOR_CLEANING, info, updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "markCompacted(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
+    ParameterizedCommand.execute(dataSourceWrapper, 

Review Comment:
   i like this style, would be even better if we declare jdbcTemplate embedding dataSourceWrapper and exposing execute command



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293071248


##########
ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java:
##########
@@ -742,12 +740,18 @@ public void testRevokeTimedOutWorkers() throws Exception {
   public void testFindPotentialCompactions() throws Exception {
     // Test that committing unlocks
     long txnid = openTxn();
-
-    List<LockComponent> components = new ArrayList<>();
-
-    components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "mytable", null, DataOperationType.UPDATE));
-    components.add(createLockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb", "yourtable", "mypartition=myvalue", DataOperationType.UPDATE));
-
+    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB,

Review Comment:
   fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] kasakrisz commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "kasakrisz (via GitHub)" <gi...@apache.org>.
kasakrisz commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1296837295


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2557,7 +2583,7 @@ public void performWriteSetGC() throws MetaException {
     }
   }
 
-  protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException, SQLException {
+  protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException {

Review Comment:
   What is `dbConn` used for? Can we achieve the same functionality with `jdbcTemplate` ?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/RetryHandler.java:
##########
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionDefinition;
+import org.springframework.transaction.support.DefaultTransactionDefinition;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * Supports building resilient APIs by automatically retrying failed function calls.
+ */
+public class RetryHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+
+
+  private DataSourceWrapper jdbcTemplate;
+  private long deadlockRetryInterval;
+  private long retryInterval;
+  private int retryLimit;
+  
+  private Configuration conf;
+  
+  public static String getMessage(Exception ex) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(ex.getMessage());
+    if (ex instanceof SQLException) {
+      SQLException sqlEx = (SQLException)ex; 
+      sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")");
+    }
+    return sb.toString();
+  }
+  
+  public void init(Configuration conf, DataSourceWrapper jdbcTemplate){
+    this.conf = conf;
+    this.jdbcTemplate = jdbcTemplate;
+
+    retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+  }
+
+  /**
+   * Establishes a {@link DataSourceWrapper.RetryContext} and executes the passed {@link TransactionalFunction}. 
+   * Automatically retries the execution in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, retryLimit, ALLOWED_REPEATED_DEADLOCKS);
+  }
+
+  /**
+   * Executes the passed {@link TransactionalFunction}, without retry in case of failure.
+   * @param properties {@link RetryCallProperties} instance used to fine-tune/alter the execution-mechanism.
+   * @param function The {@link TransactionalFunction} to execute.
+   * @return Returns with the result of the execution of the passed {@link TransactionalFunction}.
+   * @param <Result> Type of the result
+   * @throws MetaException Thrown in case of execution error.
+   */
+  public <Result> Result executeWithoutRetry(RetryCallProperties properties, TransactionalFunction<Result> function) 
+      throws MetaException {
+    return executeWithRetryInternal(properties, function, 0, 0);
+  }
+
+  private <Result> Result executeWithRetryInternal(RetryCallProperties properties, TransactionalFunction<Result> function
+      , int retryCount, int deadlockCount) throws MetaException {
+    Objects.requireNonNull(function, "RetryFunction<Result> cannot be null!");
+    Objects.requireNonNull(properties, "RetryCallProperties cannot be null!");
+
+    if (jdbcTemplate.hasRetryContext() && properties.getRetryPropagation().canJoinContext()) {

Review Comment:
   There seems to be a few combinations of retry propagations. Could you please create unit tests to cover these? WDYT?
   
   ```
   case #1:
   @Retry     @Retry(retryPropagation=CREATE_OR_JOIN)
   method1 -> calledMethod
   
   case #2:
   @Retry     @Retry(retryPropagation=CREATE_OR_NESTED)
   method1 -> calledMethod
   ...
   case #n:
              @Retry(retryPropagation=JOIN_EXISTING)
   method1 -> calledMethod
   ```



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java:
##########
@@ -131,6 +134,21 @@ enum MUTEX_KEY {
 
   int TIMED_OUT_TXN_ABORT_BATCH_SIZE = 50000;
 
+  String POOL_TX = "txnhandler";
+  String POOL_MUTEX = "mutex";
+  String POOL_COMPACTOR = "compactor";

Review Comment:
   Can these be `static final` ?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java:
##########
@@ -79,8 +79,8 @@ public class CompactionInfo implements Comparable<CompactionInfo> {
   public Set<Long> writeIds;
   public boolean hasUncompactedAborts;
 
-  byte[] metaInfo;
-  String hadoopJobId;
+  public byte[] metaInfo;
+  public String hadoopJobId;

Review Comment:
   Should these be private fields with a public getter and setter?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/ParameterizedCommand.java:
##########
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+
+import java.util.function.Function;
+
+/**
+ * Represents a parameterized command (for exmaple an UPDATE statement) as a Spring {@link NamedParameterJdbcTemplate}
+ * style parameterized query string (for example: <b>SELECT * FROM TBL WHERE ID = :id</b>), its parameters, and a result

Review Comment:
   nit.:
   ```
   (for example: <b>UPDATE TBL SET COL1 = :value WHERE ID = :id</b>)
   ```



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java:
##########
@@ -461,9 +479,11 @@ void onRename(String oldCatName, String oldDbName, String oldTabName, String old
    * @return list of CompactionInfo structs.  These will not have id, type,
    * or runAs set since these are only potential compactions not actual ones.
    */
+  @Retry(datasource = POOL_COMPACTOR)
   @RetrySemantics.ReadOnly

Review Comment:
   What does `@RetrySemantics.ReadOnly` means? Is it possible to replace with the new `RetryHanlder` introduced by this patch?
   Or is this related to hs2 - hms communication?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/impl/AbortTxnInfoHandler.java:
##########
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.impl;
+
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.TxnStatus;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.hive.metastore.txn.retryhandling.QueryHandler;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.hive.metastore.txn.TxnStore.READY_FOR_CLEANING;
+import static org.apache.hadoop.hive.metastore.txn.TxnUtils.getEpochFn;
+
+public class AbortTxnInfoHandler implements QueryHandler<List<CompactionInfo>> {
+
+  // Three inner sub-queries which are under left-join to fetch the required data for aborted txns.
+  //language=SQL
+  private static final String SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY =
+      " \"res1\".\"TC_DATABASE\" AS \"DB\", \"res1\".\"TC_TABLE\" AS \"TBL\", \"res1\".\"TC_PARTITION\" AS \"PART\", " +
+          " \"res1\".\"MIN_TXN_START_TIME\" AS \"MIN_TXN_START_TIME\", \"res1\".\"ABORTED_TXN_COUNT\" AS \"ABORTED_TXN_COUNT\", " +
+          " \"res2\".\"MIN_OPEN_WRITE_TXNID\" AS \"MIN_OPEN_WRITE_TXNID\", \"res3\".\"RETRY_RETENTION\" AS \"RETRY_RETENTION\", " +
+          " \"res3\".\"ID\" AS \"RETRY_CQ_ID\" " +
+          " FROM " +
+          // First sub-query - Gets the aborted txns with min txn start time, number of aborted txns
+          // for corresponding db, table, partition.
+          " ( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TXN_STARTED\") AS \"MIN_TXN_START_TIME\", " +
+          " COUNT(*) AS \"ABORTED_TXN_COUNT\" FROM \"TXNS\", \"TXN_COMPONENTS\" " +
+          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = :abortedState" +
+          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" %s ) \"res1\" " +
+          " LEFT JOIN" +
+          // Second sub-query - Gets the min open txn id for corresponding db, table, partition.
+          "( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TC_TXNID\") AS \"MIN_OPEN_WRITE_TXNID\" " +
+          " FROM \"TXNS\", \"TXN_COMPONENTS\" " +
+          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = :openState" +
+          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" ) \"res2\"" +
+          " ON \"res1\".\"TC_DATABASE\" = \"res2\".\"TC_DATABASE\"" +
+          " AND \"res1\".\"TC_TABLE\" = \"res2\".\"TC_TABLE\"" +
+          " AND (\"res1\".\"TC_PARTITION\" = \"res2\".\"TC_PARTITION\" " +
+          " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res2\".\"TC_PARTITION\" IS NULL)) " +
+          " LEFT JOIN " +
+          // Third sub-query - Gets the retry entries for corresponding db, table, partition.
+          "( SELECT \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", MAX(\"CQ_ID\") AS \"ID\", " +
+          " MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RETENTION\", " +
+          " MIN(\"CQ_COMMIT_TIME\") - %s + MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RECORD_CHECK\" FROM \"COMPACTION_QUEUE\" " +
+          " WHERE \"CQ_TYPE\" = :type" +
+          " GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"res3\" " +
+          " ON \"res1\".\"TC_DATABASE\" = \"res3\".\"CQ_DATABASE\" " +
+          " AND \"res1\".\"TC_TABLE\" = \"res3\".\"CQ_TABLE\" " +
+          " AND (\"res1\".\"TC_PARTITION\" = \"res3\".\"CQ_PARTITION\" " +
+          " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res3\".\"CQ_PARTITION\" IS NULL))" +
+          " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL";
+
+  private final long abortedTimeThreshold;
+  private final int abortedThreshold;
+  private final int fetchSize;
+  
+  public String getParameterizedQueryString(DatabaseProduct dbProduct) throws MetaException {
+    return dbProduct.addLimitClause(
+        fetchSize,
+        String.format(AbortTxnInfoHandler.SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY,
+            abortedTimeThreshold >= 0 ? "" : " HAVING COUNT(*) > " + abortedThreshold, getEpochFn(dbProduct)));
+  }
+
+  @Override
+  public SqlParameterSource getQueryParameters() {
+    return new MapSqlParameterSource()
+        .addValue("abortedState", TxnStatus.ABORTED.getSqlConst(), Types.CHAR)
+        .addValue("openState", TxnStatus.OPEN.getSqlConst(), Types.CHAR)
+        .addValue("type", Character.toString(TxnStore.ABORT_TXN_CLEANUP_TYPE), Types.CHAR);
+  }
+
+  @Override
+  public List<CompactionInfo> extractData(ResultSet rs) throws DataAccessException, SQLException {
+    List<CompactionInfo> readyToCleanAborts = new ArrayList<>();
+    long systemTime = System.currentTimeMillis();
+    boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
+    while (rs.next()) {
+      boolean pastTimeThreshold =
+          checkAbortedTimeThreshold && rs.getLong("MIN_TXN_START_TIME") + abortedTimeThreshold < systemTime;
+      int numAbortedTxns = rs.getInt("ABORTED_TXN_COUNT");
+      if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
+        CompactionInfo info = new CompactionInfo();
+        info.dbname = rs.getString("DB");
+        info.tableName = rs.getString("TBL");
+        info.partName = rs.getString("PART");
+        // In this case, this field contains min open write txn ID.
+        long value = rs.getLong("MIN_OPEN_WRITE_TXNID");
+        info.minOpenWriteTxnId = value > 0 ? value : Long.MAX_VALUE;
+        // The specific type, state assigned to abort cleanup.
+        info.type = CompactionType.ABORT_TXN_CLEANUP;
+        info.state = READY_FOR_CLEANING;
+        info.retryRetention = rs.getLong("RETRY_RETENTION");
+        info.id = rs.getLong("RETRY_CQ_ID");
+        readyToCleanAborts.add(info);

Review Comment:
   I assume this logic was copied from somewhere else but is it possible to switch to a streaming solution instead of load all records to memory? It should be addressed in another patch.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1708237566

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [66 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293099638


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -58,114 +68,18 @@
  * methods are not available through the thrift interface.
  */
 class CompactionTxnHandler extends TxnHandler {
-  private static final String CLASS_NAME = CompactionTxnHandler.class.getName();
-  private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
-  private static final String DB_FAILED_TO_CONNECT = "Unable to connect to transaction database: ";
-
-  private static DataSource connPoolCompaction;
-
-  private static final String SELECT_COMPACTION_QUEUE_BY_TXN_ID =
-      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
-          + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
-          + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
-          + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
-          + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
-          + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" "
-          + "FROM \"COMPACTION_QUEUE\" WHERE \"CQ_TXN_ID\" = ?";
-  private static final String SELECT_COMPACTION_METRICS_CACHE_QUERY =
-      "SELECT \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM \"COMPACTION_METRICS_CACHE\" " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ?";
-  private static final String NO_SELECT_COMPACTION_METRICS_CACHE_FOR_TYPE_QUERY =
-      "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_VALUE\", \"CMC_VERSION\" FROM " +
-      "\"COMPACTION_METRICS_CACHE\" WHERE \"CMC_METRIC_TYPE\" = ? ORDER BY \"CMC_METRIC_VALUE\" DESC";
-  private static final String UPDATE_COMPACTION_METRICS_CACHE_QUERY =
-      "UPDATE \"COMPACTION_METRICS_CACHE\" SET \"CMC_METRIC_VALUE\" = ?, \"CMC_VERSION\" = ? " +
-          "WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? AND \"CMC_METRIC_TYPE\" = ? " +
-          "AND \"CMC_VERSION\" = ?";
-  private static final String INSERT_COMPACTION_METRICS_CACHE_QUERY =
-      "INSERT INTO \"COMPACTION_METRICS_CACHE\" ( " +
-          "\"CMC_DATABASE\", \"CMC_TABLE\", \"CMC_PARTITION\", \"CMC_METRIC_TYPE\", \"CMC_METRIC_VALUE\", " +
-          "\"CMC_VERSION\" ) VALUES (?, ?, ?, ?, ?, ?)";
-  private static final String DELETE_COMPACTION_METRICS_CACHE_QUERY =
-      "DELETE FROM \"COMPACTION_METRICS_CACHE\" WHERE \"CMC_DATABASE\" = ? AND \"CMC_TABLE\" = ? " +
-          "AND \"CMC_METRIC_TYPE\" = ?";
-
-  private static final String DELETE_FAILED_TXNS_SQL =
-      "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
-          "AND (\"TXN_STATE\" = " + TxnStatus.ABORTED + " OR \"TXN_STATE\" = " + TxnStatus.COMMITTED + ") " +
-          "AND \"TXN_ID\" < ?";
-
-  // Three inner sub-queries which are under left-join to fetch the required data for aborted txns.
-  private static final String SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY =
-          " \"res1\".\"TC_DATABASE\" AS \"DB\", \"res1\".\"TC_TABLE\" AS \"TBL\", \"res1\".\"TC_PARTITION\" AS \"PART\", " +
-          " \"res1\".\"MIN_TXN_START_TIME\" AS \"MIN_TXN_START_TIME\", \"res1\".\"ABORTED_TXN_COUNT\" AS \"ABORTED_TXN_COUNT\", " +
-          " \"res2\".\"MIN_OPEN_WRITE_TXNID\" AS \"MIN_OPEN_WRITE_TXNID\", \"res3\".\"RETRY_RETENTION\" AS \"RETRY_RETENTION\", " +
-          " \"res3\".\"ID\" AS \"RETRY_CQ_ID\" " +
-      " FROM " +
-          // First sub-query - Gets the aborted txns with min txn start time, number of aborted txns
-          // for corresponding db, table, partition.
-          " ( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TXN_STARTED\") AS \"MIN_TXN_START_TIME\", " +
-          " COUNT(*) AS \"ABORTED_TXN_COUNT\" FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.ABORTED +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" %s ) \"res1\" " +
-      " LEFT JOIN" +
-          // Second sub-query - Gets the min open txn id for corresponding db, table, partition.
-          "( SELECT \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\", MIN(\"TC_TXNID\") AS \"MIN_OPEN_WRITE_TXNID\" " +
-          " FROM \"TXNS\", \"TXN_COMPONENTS\" " +
-          " WHERE \"TXN_ID\" = \"TC_TXNID\" AND \"TXN_STATE\" = " + TxnStatus.OPEN +
-          " GROUP BY \"TC_DATABASE\", \"TC_TABLE\", \"TC_PARTITION\" ) \"res2\"" +
-      " ON \"res1\".\"TC_DATABASE\" = \"res2\".\"TC_DATABASE\"" +
-      " AND \"res1\".\"TC_TABLE\" = \"res2\".\"TC_TABLE\"" +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res2\".\"TC_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res2\".\"TC_PARTITION\" IS NULL)) " +
-      " LEFT JOIN " +
-          // Third sub-query - Gets the retry entries for corresponding db, table, partition.
-          "( SELECT \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", MAX(\"CQ_ID\") AS \"ID\", " +
-          " MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RETENTION\", " +
-          " MIN(\"CQ_COMMIT_TIME\") - %s + MAX(\"CQ_RETRY_RETENTION\") AS \"RETRY_RECORD_CHECK\" FROM \"COMPACTION_QUEUE\" " +
-          " WHERE \"CQ_TYPE\" = " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"res3\" " +
-      " ON \"res1\".\"TC_DATABASE\" = \"res3\".\"CQ_DATABASE\" " +
-      " AND \"res1\".\"TC_TABLE\" = \"res3\".\"CQ_TABLE\" " +
-      " AND (\"res1\".\"TC_PARTITION\" = \"res3\".\"CQ_PARTITION\" " +
-              " OR (\"res1\".\"TC_PARTITION\" IS NULL AND \"res3\".\"CQ_PARTITION\" IS NULL))" +
-      " WHERE \"res3\".\"RETRY_RECORD_CHECK\" <= 0 OR \"res3\".\"RETRY_RECORD_CHECK\" IS NULL";
-
-  private static final String DELETE_CQ_ENTRIES = "DELETE FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = ?";
-
-  private static final String DELETE_CQ_AND_ABORT_ENTRIES = DELETE_CQ_ENTRIES +
-      " OR ( \"CQ_DATABASE\" = ? AND \"CQ_TABLE\" = ? " +
-      " AND \"CQ_PARTITION\" %s AND \"CQ_TYPE\" = " +
-      quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) + " )";
-
-  private static final String INSERT_ABORT_RETRY_ENTRY_INTO_COMPACTION_QUEUE =
-      "INSERT INTO \"COMPACTION_QUEUE\" (\"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", " +
-      " \"CQ_TYPE\", \"CQ_STATE\", \"CQ_RETRY_RETENTION\", \"CQ_ERROR_MESSAGE\", \"CQ_COMMIT_TIME\") " +
-      " VALUES (?, ?, ?, ?, ?, ?, ?, ?, %s)";
+  
+  private static final Logger LOG = LoggerFactory.getLogger(CompactionTxnHandler.class.getName());
 
   public CompactionTxnHandler() {
   }
 
-  @Override
-  public void setConf(Configuration conf) {
-    super.setConf(conf);
-    synchronized (CompactionTxnHandler.class) {
-      if (connPoolCompaction == null) {
-        int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);

Review Comment:
   where do you set the connection pool for compaction activities?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293045508


##########
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java:
##########
@@ -153,10 +153,10 @@ public void run() {
           Set<String> skipTables = Sets.newConcurrentHashSet();
 
           Set<CompactionInfo> potentials = compactionExecutor.submit(() ->
-            txnHandler.findPotentialCompactions(abortedThreshold, abortedTimeThreshold, prevStart)
-              .parallelStream()
-              .filter(ci -> isEligibleForCompaction(ci, currentCompactions, skipDBs, skipTables))
-              .collect(Collectors.toSet())).get();
+              txnHandler.findPotentialCompactions(abortedThreshold, abortedTimeThreshold, prevStart)

Review Comment:
   same here, no code changes just reformatting



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1293086349


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,38 +261,22 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
-    try {
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction)) {
-        try (PreparedStatement stmt = dbConn.prepareStatement(DELETE_FAILED_TXNS_SQL)) {
-          //Aborted and committed are terminal states, so nothing about the txn can change
-          //after that, so READ COMMITTED is sufficient.
-          /*
-           * Only delete aborted / committed transaction in a way that guarantees two things:
-           * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
-           * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
-           */
-          long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId(dbConn);
-          stmt.setLong(1, lowWaterMark);
-          LOG.debug("Going to execute query <{}>", DELETE_FAILED_TXNS_SQL);
-          int rc = stmt.executeUpdate();
-          LOG.debug("Removed {} empty Aborted and Committed transactions from TXNS", rc);
-          LOG.debug("Going to commit");
-          dbConn.commit();
-        } catch (SQLException e) {
-          LOG.error("Unable to delete from txns table " + e.getMessage());
-          LOG.debug("Going to rollback");
-          rollbackDBConn(dbConn);
-          checkRetryable(e, "cleanEmptyAbortedTxns");
-          throw new MetaException("Unable to delete from txns table " + e.getMessage());
-        }
-      } catch (SQLException e) {
-        LOG.error(DB_FAILED_TO_CONNECT + e.getMessage());
-        checkRetryable(e, "cleanEmptyAbortedTxns");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedAndCommittedTxns();
-    }
+    //Aborted and committed are terminal states, so nothing about the txn can change
+    //after that, so READ COMMITTED is sufficient.
+    /*
+     * Only delete aborted / committed transaction in a way that guarantees two things:
+     * 1. never deletes anything that is inside the TXN_OPENTXN_TIMEOUT window
+     * 2. never deletes the maximum txnId even if it is before the TXN_OPENTXN_TIMEOUT window
+     */
+    long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();
+    String query = "DELETE FROM \"TXNS\" WHERE \"TXN_ID\" NOT IN (SELECT \"TC_TXNID\" FROM \"TXN_COMPONENTS\") " +
+        "AND (\"TXN_STATE\" = :abortedState OR \"TXN_STATE\" = :committedState) AND \"TXN_ID\" < :txnId";
+
+    LOG.debug("Going to execute query <{}>", query);
+    dataSourceWrapper.getJdbcTemplate().update(query, new MapSqlParameterSource()

Review Comment:
   can't we use ParameterizedCommand here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1680354529

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [66 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1714654036

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [4 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [68 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   ![warning](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/message_warning-16px.png 'warning') The version of Java (11.0.8) you have used to run this analysis is deprecated and we will stop accepting it soon. Please update to at least Java 17.
   Read more [here](https://docs.sonarcloud.io/appendices/scanner-environment/)
   
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337111661


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,37 +262,25 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
+    //Aborted and committed are terminal states, so nothing about the txn can change

Review Comment:
   i think we can drop this comment, it doesn't make much sence



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -1001,37 +262,25 @@ public void removeDuplicateCompletedTxnComponents() throws MetaException {
   @RetrySemantics.SafeToRetry
   public void cleanEmptyAbortedAndCommittedTxns() throws MetaException {
     LOG.info("Start to clean empty aborted or committed TXNS");
+    //Aborted and committed are terminal states, so nothing about the txn can change

Review Comment:
   i think we can drop this comment, it doesn't make much sense



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337025667


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -444,136 +175,15 @@ public void markCompacted(CompactionInfo info) throws MetaException {
   @Override
   @RetrySemantics.ReadOnly
   public List<CompactionInfo> findReadyToClean(long minOpenTxnWaterMark, long retentionTime) throws MetaException {
-    try {
-      List<CompactionInfo> rc = new ArrayList<>();
-
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-          Statement stmt = dbConn.createStatement()) {
-        /*
-         * By filtering on minOpenTxnWaterMark, we will only cleanup after every transaction is committed, that could see
-         * the uncompacted deltas. This way the cleaner can clean up everything that was made obsolete by this compaction.
-         */
-        String whereClause = " WHERE \"CQ_STATE\" = " + quoteChar(READY_FOR_CLEANING) +
-          " AND \"CQ_TYPE\" != " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " AND (\"CQ_COMMIT_TIME\" < (" + getEpochFn(dbProduct) + " - \"CQ_RETRY_RETENTION\" - " + retentionTime + ") OR \"CQ_COMMIT_TIME\" IS NULL)";
-        
-        String queryStr = 
-          "  \"CQ_ID\", \"cq1\".\"CQ_DATABASE\", \"cq1\".\"CQ_TABLE\", \"cq1\".\"CQ_PARTITION\"," +
-          "  \"CQ_TYPE\", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_TBLPROPERTIES\", \"CQ_RETRY_RETENTION\", " +
-          "  \"CQ_NEXT_TXN_ID\"";
-        if (useMinHistoryWriteId) {
-          queryStr += ", \"MIN_OPEN_WRITE_ID\"";
-        }
-        queryStr +=
-          "  FROM \"COMPACTION_QUEUE\" \"cq1\" " +
-          "INNER JOIN (" +
-          "  SELECT MIN(\"CQ_HIGHEST_WRITE_ID\") \"MIN_WRITE_ID_HWM\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\"" +
-          "  FROM \"COMPACTION_QUEUE\"" 
-          + whereClause +
-          "  GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"cq2\" " +
-          "ON \"cq1\".\"CQ_DATABASE\" = \"cq2\".\"CQ_DATABASE\""+
-          "  AND \"cq1\".\"CQ_TABLE\" = \"cq2\".\"CQ_TABLE\""+
-          "  AND (\"cq1\".\"CQ_PARTITION\" = \"cq2\".\"CQ_PARTITION\"" +
-          "    OR \"cq1\".\"CQ_PARTITION\" IS NULL AND \"cq2\".\"CQ_PARTITION\" IS NULL)" +
-          "  AND \"CQ_HIGHEST_WRITE_ID\" = \"MIN_WRITE_ID_HWM\" ";
-        
-        if (useMinHistoryWriteId) {
-          queryStr += 
-            "LEFT JOIN (" +
-            "  SELECT MIN(\"MH_WRITEID\") \"MIN_OPEN_WRITE_ID\", \"MH_DATABASE\", \"MH_TABLE\"" +
-            "  FROM \"MIN_HISTORY_WRITE_ID\"" +
-            "  GROUP BY \"MH_DATABASE\", \"MH_TABLE\") \"hwm\" " +
-            "ON \"cq1\".\"CQ_DATABASE\" = \"hwm\".\"MH_DATABASE\"" +
-            "  AND \"cq1\".\"CQ_TABLE\" = \"hwm\".\"MH_TABLE\"";
-          
-          whereClause += " AND (\"CQ_HIGHEST_WRITE_ID\" < \"MIN_OPEN_WRITE_ID\" OR \"MIN_OPEN_WRITE_ID\" IS NULL)";
-          
-        } else if (minOpenTxnWaterMark > 0) {
-          whereClause += " AND (\"CQ_NEXT_TXN_ID\" <= " + minOpenTxnWaterMark + " OR \"CQ_NEXT_TXN_ID\" IS NULL)";
-        }
-
-        queryStr += whereClause + " ORDER BY \"CQ_ID\"";
-
-        queryStr = dbProduct.addLimitClause(MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE), queryStr);
-        LOG.debug("Going to execute query <{}>", queryStr);
-
-        try (ResultSet rs = stmt.executeQuery(queryStr)) {
-          while (rs.next()) {
-            CompactionInfo info = new CompactionInfo();
-            info.id = rs.getLong(1);
-            info.dbname = rs.getString(2);
-            info.tableName = rs.getString(3);
-            info.partName = rs.getString(4);
-            info.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(5).charAt(0));
-            info.runAs = rs.getString(6);
-            info.highestWriteId = rs.getLong(7);
-            info.properties = rs.getString(8);
-            info.retryRetention = rs.getInt(9);
-            info.nextTxnId = rs.getLong(10);
-            if (useMinHistoryWriteId) {
-              info.minOpenWriteId = rs.getLong(11);
-            }
-            LOG.debug("Found ready to clean: {}", info);
-            rc.add(info);
-          }
-        }
-        return rc;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
-        checkRetryable(e, "findReadyToClean");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      return findReadyToClean(minOpenTxnWaterMark, retentionTime);
-    }
+    return jdbcResource.execute(new ReadyToCleanHandler(useMinHistoryWriteId, minOpenTxnWaterMark, retentionTime, 
+        MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE)));

Review Comment:
   MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE) is repeating in multiple handlers, should we pass conf a create function to getFetchSize from conf?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1337027620


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -615,57 +190,19 @@ public void markCleanerStart(CompactionInfo info) throws MetaException {
   @RetrySemantics.ReadOnly
   public void clearCleanerStart(CompactionInfo info) throws MetaException {
     LOG.debug("Running clearCleanerStart with CompactionInfo: {}", info);
-
-    try {
-      Connection dbConn = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-        setCleanerStart(dbConn, info, -1L);
-      } catch (SQLException e) {
-        LOG.error("Unable to clear the cleaner start time for compaction record  " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(e, "clearCleanerStart(" + info + ")");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      clearCleanerStart(info);
-    }
+    setCleanerStart(info, -1L);
   }
 
-  private void setCleanerStart(Connection dbConn, CompactionInfo info, Long timestamp)
-      throws RetryException, SQLException {
-    long id = info.id;
-    PreparedStatement pStmt = null;
-    ResultSet rs = null;
-    try {
-      String query = "" +
-          " UPDATE " +
-          "  \"COMPACTION_QUEUE\" " +
-          " SET " +
-          "  \"CQ_CLEANER_START\" = " + timestamp +
-          " WHERE " +
-          "  \"CQ_ID\" = " + id +
-          " AND " +
-          "  \"CQ_STATE\"='" + READY_FOR_CLEANING + "'";
-
-      pStmt = dbConn.prepareStatement(query);
-      LOG.debug("Going to execute update <{}> for CQ_ID={}", query, id);
-      int updCount = pStmt.executeUpdate();
-      if (updCount != 1) {
-        LOG.error("Unable to update compaction record: {}.  Update count={}", info, updCount);
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-      } else {
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      }
-    } finally {
-      close(rs);
-      closeStmt(pStmt);
-    }
+  private void setCleanerStart(CompactionInfo info, Long timestamp) throws MetaException {
+    jdbcTemplate.execute(
+        " UPDATE \"COMPACTION_QUEUE\" " +

Review Comment:
   no, that's fine



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1625310219

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [3 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [86 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309101975


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -882,24 +926,15 @@ public void setOpenTxnTimeOutMillis(long openTxnTimeOutMillis) {
     TxnHandler.openTxnTimeOutMillis = openTxnTimeOutMillis;
   }
 
-  protected long getOpenTxnTimeoutLowBoundaryTxnId(Connection dbConn) throws MetaException, SQLException {
-    long maxTxnId;
-    String s =
-        "SELECT MAX(\"TXN_ID\") FROM \"TXNS\" WHERE \"TXN_STARTED\" < (" + getEpochFn(dbProduct) + " - "
-            + openTxnTimeOutMillis + ")";
-    try (Statement stmt = dbConn.createStatement()) {
-      LOG.debug("Going to execute query <{}>", s);
-      try (ResultSet maxTxnIdRs = stmt.executeQuery(s)) {
-        maxTxnIdRs.next();
-        maxTxnId = maxTxnIdRs.getLong(1);
-        if (maxTxnIdRs.wasNull()) {
-          /*
-           * TXNS always contains at least one transaction,
-           * the row where txnid = (select max(txnid) where txn_started < epoch - TXN_OPENTXN_TIMEOUT) is never deleted
-           */
-          throw new MetaException("Transaction tables not properly " + "initialized, null record found in MAX(TXN_ID)");
-        }
-      }
+  protected long getOpenTxnTimeoutLowBoundaryTxnId() throws MetaException {    
+    Long maxTxnId = jdbcTemplate.getJdbcTemplate().queryForObject("SELECT MAX(\"TXN_ID\") FROM \"TXNS\" WHERE \"TXN_STARTED\" < (" + getEpochFn(dbProduct) + " - "

Review Comment:
   would it open a new connection? The idea here was to reuse the existing connection



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309975174


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -1603,21 +1645,24 @@ public void commitTxn(CommitTxnRequest rqst) throws NoSuchTxnException, TxnAbort
         }
 
         if (!isHiveReplTxn) {
-          createCommitNotificationEvent(dbConn, txnid , txnType);
+          createCommitNotificationEvent(jdbcTemplate, txnid , txnType);

Review Comment:
   done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309810514


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java:
##########
@@ -405,6 +410,50 @@ public static List<Integer> buildQueryWithINClauseStrings(Configuration conf, Li
     return ret;
   }
 
+  /**
+   * Executes the statement with an IN clause. If the number of elements or the length of the constructed statement would be
+   * too big, the IN clause will be split into multiple smaller ranges, and the statement will be executed multiple times.
+   * @param conf Hive configuration used to get the query and IN clause length limits.
+   * @param jdbcTemplate The {@link NamedParameterJdbcTemplate} instance to used for statement execution.
+   * @param query The query with the IN clause
+   * @param params A {@link MapSqlParameterSource} instance with the parameters of the query
+   * @param inClauseParamName The name of the parameter representing the content of the IN clause 
+   * @param elements A {@link List} containing the elements to put in the IN clause
+   * @param comparator A {@link Comparator} instance used to find the longest element in the list. Used to
+   *                   estimate the length of the query.
+   * @return Returns the total number of affected rows.
+   * @param <T> Type of the elements in the list.
+   */
+  public static <T> int executeStatementWithInClause(Configuration conf, NamedParameterJdbcTemplate jdbcTemplate, 

Review Comment:
   shouldn't this be part of util class?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309884737


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/retryhandling/ParameterizedCommand.java:
##########
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.retryhandling;
+
+import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
+
+import java.util.function.Function;
+
+/**
+ * Represents a parameterized command (for exmaple an UPDATE statement) as a Spring {@link NamedParameterJdbcTemplate}
+ * style parameterized query string (for example: <b>SELECT * FROM TBL WHERE ID = :id</b>), its parameters, and a result

Review Comment:
   fixed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309913155


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -378,15 +393,21 @@ public void setConf(Configuration conf){
     int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
     synchronized (TxnHandler.class) {
       try (DataSourceProvider.DataSourceNameConfigurator configurator =
-               new DataSourceProvider.DataSourceNameConfigurator(conf, "txnhandler")) {
+               new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_TX)) {
         if (connPool == null) {
           connPool = setupJdbcConnectionPool(conf, maxPoolSize);
         }
         if (connPoolMutex == null) {
-          configurator.resetName("mutex");
+          configurator.resetName(POOL_MUTEX);
           connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize);
         }
       }
+      maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.HIVE_COMPACTOR_CONNECTION_POOLING_MAX_CONNECTIONS);
+      try (DataSourceProvider.DataSourceNameConfigurator ignored = new DataSourceProvider.DataSourceNameConfigurator(conf, POOL_COMPACTOR)) {

Review Comment:
   Fixed



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1309918630


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java:
##########
@@ -444,136 +158,15 @@ public void markCompacted(CompactionInfo info) throws MetaException {
   @Override
   @RetrySemantics.ReadOnly
   public List<CompactionInfo> findReadyToClean(long minOpenTxnWaterMark, long retentionTime) throws MetaException {
-    try {
-      List<CompactionInfo> rc = new ArrayList<>();
-
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-          Statement stmt = dbConn.createStatement()) {
-        /*
-         * By filtering on minOpenTxnWaterMark, we will only cleanup after every transaction is committed, that could see
-         * the uncompacted deltas. This way the cleaner can clean up everything that was made obsolete by this compaction.
-         */
-        String whereClause = " WHERE \"CQ_STATE\" = " + quoteChar(READY_FOR_CLEANING) +
-          " AND \"CQ_TYPE\" != " + quoteChar(TxnStore.ABORT_TXN_CLEANUP_TYPE) +
-          " AND (\"CQ_COMMIT_TIME\" < (" + getEpochFn(dbProduct) + " - \"CQ_RETRY_RETENTION\" - " + retentionTime + ") OR \"CQ_COMMIT_TIME\" IS NULL)";
-        
-        String queryStr = 
-          "  \"CQ_ID\", \"cq1\".\"CQ_DATABASE\", \"cq1\".\"CQ_TABLE\", \"cq1\".\"CQ_PARTITION\"," +
-          "  \"CQ_TYPE\", \"CQ_RUN_AS\", \"CQ_HIGHEST_WRITE_ID\", \"CQ_TBLPROPERTIES\", \"CQ_RETRY_RETENTION\", " +
-          "  \"CQ_NEXT_TXN_ID\"";
-        if (useMinHistoryWriteId) {
-          queryStr += ", \"MIN_OPEN_WRITE_ID\"";
-        }
-        queryStr +=
-          "  FROM \"COMPACTION_QUEUE\" \"cq1\" " +
-          "INNER JOIN (" +
-          "  SELECT MIN(\"CQ_HIGHEST_WRITE_ID\") \"MIN_WRITE_ID_HWM\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\"" +
-          "  FROM \"COMPACTION_QUEUE\"" 
-          + whereClause +
-          "  GROUP BY \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\") \"cq2\" " +
-          "ON \"cq1\".\"CQ_DATABASE\" = \"cq2\".\"CQ_DATABASE\""+
-          "  AND \"cq1\".\"CQ_TABLE\" = \"cq2\".\"CQ_TABLE\""+
-          "  AND (\"cq1\".\"CQ_PARTITION\" = \"cq2\".\"CQ_PARTITION\"" +
-          "    OR \"cq1\".\"CQ_PARTITION\" IS NULL AND \"cq2\".\"CQ_PARTITION\" IS NULL)" +
-          "  AND \"CQ_HIGHEST_WRITE_ID\" = \"MIN_WRITE_ID_HWM\" ";
-        
-        if (useMinHistoryWriteId) {
-          queryStr += 
-            "LEFT JOIN (" +
-            "  SELECT MIN(\"MH_WRITEID\") \"MIN_OPEN_WRITE_ID\", \"MH_DATABASE\", \"MH_TABLE\"" +
-            "  FROM \"MIN_HISTORY_WRITE_ID\"" +
-            "  GROUP BY \"MH_DATABASE\", \"MH_TABLE\") \"hwm\" " +
-            "ON \"cq1\".\"CQ_DATABASE\" = \"hwm\".\"MH_DATABASE\"" +
-            "  AND \"cq1\".\"CQ_TABLE\" = \"hwm\".\"MH_TABLE\"";
-          
-          whereClause += " AND (\"CQ_HIGHEST_WRITE_ID\" < \"MIN_OPEN_WRITE_ID\" OR \"MIN_OPEN_WRITE_ID\" IS NULL)";
-          
-        } else if (minOpenTxnWaterMark > 0) {
-          whereClause += " AND (\"CQ_NEXT_TXN_ID\" <= " + minOpenTxnWaterMark + " OR \"CQ_NEXT_TXN_ID\" IS NULL)";
-        }
-
-        queryStr += whereClause + " ORDER BY \"CQ_ID\"";
-
-        queryStr = dbProduct.addLimitClause(MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE), queryStr);
-        LOG.debug("Going to execute query <{}>", queryStr);
-
-        try (ResultSet rs = stmt.executeQuery(queryStr)) {
-          while (rs.next()) {
-            CompactionInfo info = new CompactionInfo();
-            info.id = rs.getLong(1);
-            info.dbname = rs.getString(2);
-            info.tableName = rs.getString(3);
-            info.partName = rs.getString(4);
-            info.type = TxnUtils.dbCompactionType2ThriftType(rs.getString(5).charAt(0));
-            info.runAs = rs.getString(6);
-            info.highestWriteId = rs.getLong(7);
-            info.properties = rs.getString(8);
-            info.retryRetention = rs.getInt(9);
-            info.nextTxnId = rs.getLong(10);
-            if (useMinHistoryWriteId) {
-              info.minOpenWriteId = rs.getLong(11);
-            }
-            LOG.debug("Found ready to clean: {}", info);
-            rc.add(info);
-          }
-        }
-        return rc;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
-        checkRetryable(e, "findReadyToClean");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      return findReadyToClean(minOpenTxnWaterMark, retentionTime);
-    }
+    return jdbcTemplate.execute(new ReadyToCleanHandler(useMinHistoryWriteId, minOpenTxnWaterMark, retentionTime, 
+        MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE)));
   }
 
   @Override
   @RetrySemantics.ReadOnly
   public List<CompactionInfo> findReadyToCleanAborts(long abortedTimeThreshold, int abortedThreshold) throws MetaException {
-    try {
-      List<CompactionInfo> readyToCleanAborts = new ArrayList<>();
-      try (Connection dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED, connPoolCompaction);
-           Statement stmt = dbConn.createStatement()) {
-        boolean checkAbortedTimeThreshold = abortedTimeThreshold >= 0;
-
-        String sCheckAborted = dbProduct.addLimitClause(
-                MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_FETCH_SIZE),
-                String.format(SELECT_ABORTS_WITH_MIN_OPEN_WRITETXN_QUERY,
-                checkAbortedTimeThreshold ? "" : " HAVING COUNT(*) > " + abortedThreshold, getEpochFn(dbProduct)));
-        LOG.debug("Going to execute query <{}>", sCheckAborted);
-
-        try (ResultSet rs = stmt.executeQuery(sCheckAborted)) {
-          long systemTime = System.currentTimeMillis();
-          while (rs.next()) {
-            boolean pastTimeThreshold =
-                    checkAbortedTimeThreshold && rs.getLong(4) + abortedTimeThreshold < systemTime;
-            int numAbortedTxns = rs.getInt(5);
-            if (numAbortedTxns > abortedThreshold || pastTimeThreshold) {
-              CompactionInfo info = new CompactionInfo();
-              info.dbname = rs.getString(1);
-              info.tableName = rs.getString(2);
-              info.partName = rs.getString(3);
-              // In this case, this field contains min open write txn ID.
-              info.minOpenWriteTxnId = rs.getLong(6) > 0 ? rs.getLong(6) : Long.MAX_VALUE;
-              // The specific type, state assigned to abort cleanup.
-              info.type = CompactionType.ABORT_TXN_CLEANUP;
-              info.state = READY_FOR_CLEANING;
-              info.retryRetention = rs.getLong(7);
-              info.id = rs.getLong(8);
-              readyToCleanAborts.add(info);
-            }
-          }
-        }
-        return readyToCleanAborts;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
-        checkRetryable(e, "findReadyToCleanAborts");
-        throw new MetaException(DB_FAILED_TO_CONNECT + e.getMessage());
-      }
-    } catch (RetryException e) {
-      return findReadyToCleanAborts(abortedTimeThreshold, abortedThreshold);
-    }
+    return jdbcTemplate.execute(new AbortTxnInfoHandler(abortedTimeThreshold, abortedThreshold,

Review Comment:
   done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310007234


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -473,6 +505,18 @@ private boolean checkIfTableIsUsable(String tableName, boolean configValue) thro
     }
     return tableExists;
   }
+  
+  @Override
+  @RetrySemantics.ReadOnly
+  public DatabaseProduct getDatabaseProduct() {

Review Comment:
   Good catch! This is a remnant of an earlier approach, removed.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310060710


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2575,11 +2601,17 @@ protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException,
           minOpenTxn = Long.MAX_VALUE;
         }
       }
+    } catch (SQLException e) {
+      throw new UncategorizedSQLException(null, null, e);
+    }
+    jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);
+    try {
+      long lowWaterMark = getOpenTxnTimeoutLowBoundaryTxnId();

Review Comment:
   do we open a new connection here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] deniskuzZ commented on a diff in pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "deniskuzZ (via GitHub)" <gi...@apache.org>.
deniskuzZ commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1310059255


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java:
##########
@@ -2575,11 +2601,17 @@ protected long getMinOpenTxnIdWaterMark(Connection dbConn) throws MetaException,
           minOpenTxn = Long.MAX_VALUE;
         }
       }
+    } catch (SQLException e) {
+      throw new UncategorizedSQLException(null, null, e);
+    }
+    jdbcTemplate.getTransactionWithinRetryContext(new DefaultTransactionDefinition(TransactionDefinition.PROPAGATION_REQUIRED), POOL_TX);

Review Comment:
   do we open a new connection here?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] rkirtir commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "rkirtir (via GitHub)" <gi...@apache.org>.
rkirtir commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1217874533


##########
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java:
##########
@@ -3496,13 +3497,15 @@ public void testAbortCompaction() throws Exception {
     // Verify  compaction order
     List<ShowCompactResponseElement> compacts =
             txnHandler.showCompact(new ShowCompactRequest()).getCompacts();
+    compacts.sort(Comparator.comparingLong(ShowCompactResponseElement::getId));
+    
     Assert.assertEquals(6, compacts.size());
-    Assert.assertEquals(TxnStore.INITIATED_RESPONSE, compacts.get(0).getState());
-    Assert.assertEquals(TxnStore.REFUSED_RESPONSE, compacts.get(1).getState());
+    Assert.assertEquals(TxnStore.REFUSED_RESPONSE, compacts.get(0).getState());
+    Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, compacts.get(1).getState());
     Assert.assertEquals(TxnStore.CLEANING_RESPONSE, compacts.get(2).getState());
     Assert.assertEquals(TxnStore.CLEANING_RESPONSE, compacts.get(3).getState());
-    Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, compacts.get(4).getState());
-    Assert.assertEquals(TxnStore.REFUSED_RESPONSE, compacts.get(5).getState());
+    Assert.assertEquals(TxnStore.REFUSED_RESPONSE, compacts.get(4).getState());
+    Assert.assertEquals(TxnStore.INITIATED_RESPONSE, compacts.get(5).getState());

Review Comment:
   Seems default sorting order of show compaction has compromised



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1617647399

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [80 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: HIVE-27406: CompactionTxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1618610818

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [80 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1576571368

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [33 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] veghlaci05 commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "veghlaci05 (via GitHub)" <gi...@apache.org>.
veghlaci05 commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1222917921


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/RetryHandler.java:
##########
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.UncategorizedSQLException;
+import org.springframework.transaction.TransactionStatus;
+import org.springframework.transaction.support.TransactionTemplate;
+
+import java.sql.SQLException;
+import java.util.Objects;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+public class RetryHandler {
+
+  @FunctionalInterface
+  public interface VoidRetryFunction {
+
+    void call(TransactionStatus status) throws SQLException, MetaException;
+
+  }
+
+  @FunctionalInterface
+  public interface RetryFunction<Result> {
+
+    Result call(TransactionStatus status) throws SQLException, MetaException;
+
+  }
+
+  private static final Logger LOG = LoggerFactory.getLogger(RetryHandler.class);
+
+  private static final int ALLOWED_REPEATED_DEADLOCKS = 10;
+  private static final String MANUAL_RETRY = "ManualRetry";
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+
+
+  private DatabaseProduct dbProduct;
+
+  private long deadlockRetryInterval;
+  private long retryInterval;
+
+  private int deadlockCnt;
+  private int retryLimit;
+  private int retryNum;
+  
+  private Configuration conf;
+
+
+  public static String getMessage(Exception ex) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(ex.getMessage());
+    if (ex instanceof SQLException) {
+      SQLException sqlEx = (SQLException)ex; 
+      sb.append(" (SQLState=").append(sqlEx.getSQLState()).append(", ErrorCode=").append(sqlEx.getErrorCode()).append(")");
+    }
+    return sb.toString();
+  }  
+  
+  public void init(Configuration conf, DatabaseProduct dbProduct){
+    this.conf = conf;
+    this.dbProduct = dbProduct;
+
+    retryInterval = MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_INTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, MetastoreConf.ConfVars.HMS_HANDLER_ATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+  }
+  
+  public <Result> Result executeWithRetry(CallProperties properties, RetryFunction<Result> function) throws MetaException {
+    return executeWithRetryInternal(properties, function, true);
+  }
+
+  public void executeWithRetry(CallProperties properties, VoidRetryFunction function) throws MetaException {
+    executeWithRetryInternal(properties, (TransactionStatus status) -> { 
+      function.call(status); 
+      return null;
+    }, true);
+  }
+
+  public  <Result> Result executeWithoutRetry(CallProperties properties, RetryFunction<Result> function) throws MetaException {
+    return executeWithRetryInternal(properties, (TransactionStatus status) -> {
+      function.call(status);
+      return null;

Review Comment:
   Good catch, this was a copy-paste issue.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] InvisibleProgrammer commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "InvisibleProgrammer (via GitHub)" <gi...@apache.org>.
InvisibleProgrammer commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1226514750


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state
+    try {

Review Comment:
   Can that kind of extra work with error handling stuff extracted into a private method to make the code easier to read? 



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);

Review Comment:
   I think it is really hard to understand when we read row data from resultset by column position indexes. Instead of that, I recommend using column naming, like `fullCi.id = rs.getLong("id");`.
   
   With column indexes, you can easily make a mistake by referencing wrong column. And also, changing the resultset, like adding a new third column and sliding the others right, can be tedious. 
   



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {

Review Comment:
   What is the reason of having a setter that returns with the object?



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/SqlParamSourceMapper.java:
##########
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+public interface SqlParamSourceMapper<T> {

Review Comment:
   What is the intention of that interface? 



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state

Review Comment:
   Potential null pointer exception. 



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CallProperties.java:
##########
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.datanucleus.transaction.TransactionIsolation;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.sql.Connection;
+import java.util.function.Function;
+
+public class CallProperties {

Review Comment:
   What is the intention of that class?



##########
standalone-metastore/metastore-server/pom.xml:
##########
@@ -23,6 +23,7 @@
   <name>Hive Metastore Server</name>
   <properties>
     <standalone.metastore.path.to.root>..</standalone.metastore.path.to.root>
+    <spring.jdbc.version>5.2.24.RELEASE</spring.jdbc.version>

Review Comment:
   Version info should be in root pom



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] sonarcloud[bot] commented on pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "sonarcloud[bot] (via GitHub)" <gi...@apache.org>.
sonarcloud[bot] commented on PR #4384:
URL: https://github.com/apache/hive/pull/4384#issuecomment-1587898843

   Kudos, SonarCloud Quality Gate passed!&nbsp; &nbsp; [![Quality Gate passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=4384)
   
   [![Bug](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/bug-16px.png 'Bug')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [![C](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/C-16px.png 'C')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG) [2 Bugs](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=BUG)  
   [![Vulnerability](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/vulnerability-16px.png 'Vulnerability')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=VULNERABILITY)  
   [![Security Hotspot](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/security_hotspot-16px.png 'Security Hotspot')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=4384&resolved=false&types=SECURITY_HOTSPOT)  
   [![Code Smell](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/code_smell-16px.png 'Code Smell')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [![A](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/RatingBadge/A-16px.png 'A')](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL) [35 Code Smells](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=4384&resolved=false&types=CODE_SMELL)
   
   [![No Coverage information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/CoverageChart/NoCoverageInfo-16px.png 'No Coverage information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=coverage&view=list) No Coverage information  
   [![No Duplication information](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/Duplications/NoDuplicationInfo-16px.png 'No Duplication information')](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=4384&metric=duplicated_lines_density&view=list) No Duplication information
   
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] InvisibleProgrammer commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "InvisibleProgrammer (via GitHub)" <gi...@apache.org>.
InvisibleProgrammer commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1230710212


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {

Review Comment:
   Ok, thx.



##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CallProperties.java:
##########
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.datanucleus.transaction.TransactionIsolation;
+import org.springframework.transaction.PlatformTransactionManager;
+
+import java.sql.Connection;
+import java.util.function.Function;
+
+public class CallProperties {

Review Comment:
   Ok, thx



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org


[GitHub] [hive] InvisibleProgrammer commented on a diff in pull request #4384: [Draft] HIVE-27406: CompactionTxnHandler and TxnHandler cleanup

Posted by "InvisibleProgrammer (via GitHub)" <gi...@apache.org>.
InvisibleProgrammer commented on code in PR #4384:
URL: https://github.com/apache/hive/pull/4384#discussion_r1230710832


##########
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/entitymappers/CompactionInfoMapper.java:
##########
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn.entitymappers;
+
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
+import org.apache.hadoop.hive.metastore.txn.MetaWrapperException;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.springframework.jdbc.core.RowMapper;
+import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
+import org.springframework.jdbc.core.namedparam.SqlParameterSource;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+public class CompactionInfoMapper implements RowMapper<CompactionInfo>, SqlParamSourceMapper<CompactionInfo> {
+  
+  private long compactionEndTime = 0L;
+
+  // language=PostgreSQL
+  public static final String SELECT = 
+      "SELECT \"CQ_ID\", \"CQ_DATABASE\", \"CQ_TABLE\", \"CQ_PARTITION\", "
+      + "\"CQ_STATE\", \"CQ_TYPE\", \"CQ_TBLPROPERTIES\", \"CQ_WORKER_ID\", \"CQ_START\", \"CQ_RUN_AS\", "
+      + "\"CQ_HIGHEST_WRITE_ID\", \"CQ_META_INFO\", \"CQ_HADOOP_JOB_ID\", \"CQ_ERROR_MESSAGE\", "
+      + "\"CQ_ENQUEUE_TIME\", \"CQ_WORKER_VERSION\", \"CQ_INITIATOR_ID\", \"CQ_INITIATOR_VERSION\", "
+      + "\"CQ_RETRY_RETENTION\", \"CQ_NEXT_TXN_ID\", \"CQ_TXN_ID\", \"CQ_COMMIT_TIME\", \"CQ_POOL_NAME\", "
+      + "\"CQ_NUMBER_OF_BUCKETS\", \"CQ_ORDER_BY\" FROM \"COMPACTION_QUEUE\" WHERE \"CQ_ID\" = :id";
+
+  // language=PostgreSQL
+  public static final String INSERT =
+      "INSERT INTO \"COMPLETED_COMPACTIONS\" " +
+          "   (\"CC_ID\", \"CC_DATABASE\", \"CC_TABLE\", \"CC_PARTITION\", \"CC_STATE\", \"CC_TYPE\", " +
+          "   \"CC_TBLPROPERTIES\", \"CC_WORKER_ID\", \"CC_START\", \"CC_END\", \"CC_RUN_AS\", " +
+          "   \"CC_HIGHEST_WRITE_ID\", \"CC_META_INFO\", \"CC_HADOOP_JOB_ID\", \"CC_ERROR_MESSAGE\", " +
+          "   \"CC_ENQUEUE_TIME\", \"CC_WORKER_VERSION\", \"CC_INITIATOR_ID\", \"CC_INITIATOR_VERSION\"," +
+          "   \"CC_NEXT_TXN_ID\", \"CC_TXN_ID\", \"CC_COMMIT_TIME\", \"CC_POOL_NAME\", \"CC_NUMBER_OF_BUCKETS\", " +
+          "   \"CC_ORDER_BY\") " +
+          "   VALUES(:id,:dbname,:tableName,:partName,:state,:type,:properties,:workerId,:start,:endTime," +
+          "    :runAs,:highestWriteId,:metaInfo,:hadoopJobId,:errorMessage,:enqueueTime,:workerVersion,:initiatorId," +
+          "    :initiatorVersion,:nextTxnId,:txnId,:commitTime,:poolName,:numberOfBuckets,:orderByClause)";
+
+
+  public long getCompactionEndTime() {
+    return compactionEndTime;
+  }
+
+  public CompactionInfoMapper setCompactionEndTime(long compactionEndTime) {
+    this.compactionEndTime = compactionEndTime;
+    return this;
+  }
+
+  @Override
+  public CompactionInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state

Review Comment:
   Ok, thx.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org