You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pd...@apache.org on 2024/09/09 16:25:43 UTC

(zeppelin) branch master updated: [ZEPPELIN-6080] Add support for bq single region dataset query (#4815)

This is an automated email from the ASF dual-hosted git repository.

pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git


The following commit(s) were added to refs/heads/master by this push:
     new 205014f0e4 [ZEPPELIN-6080] Add support for bq single region dataset query (#4815)
205014f0e4 is described below

commit 205014f0e4f4abda7a833627c246dd522fb41756
Author: meharanjan318 <45...@users.noreply.github.com>
AuthorDate: Mon Sep 9 21:55:37 2024 +0530

    [ZEPPELIN-6080] Add support for bq single region dataset query (#4815)
    
    * Add support for bq single region dataset query
    
    * Remove duplicate code
    
    * Resolve review comments
    
    * Updated documentation
    
    * Improved region check for bq interpreter
---
 bigquery/pom.xml                                            |  4 ++++
 .../org/apache/zeppelin/bigquery/BigQueryInterpreter.java   | 13 ++++++++++---
 bigquery/src/main/resources/interpreter-setting.json        |  7 +++++++
 docs/interpreter/bigquery.md                                |  5 +++++
 4 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/bigquery/pom.xml b/bigquery/pom.xml
index 3a0c8ab64a..a101374d87 100644
--- a/bigquery/pom.xml
+++ b/bigquery/pom.xml
@@ -76,6 +76,10 @@
       <artifactId>guava</artifactId>
       <version>${guava.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
 
   </dependencies>
 
diff --git a/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java b/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
index f80a933b6e..c23bd228e7 100644
--- a/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
+++ b/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
@@ -37,6 +37,7 @@ import com.google.api.services.bigquery.model.TableFieldSchema;
 import com.google.api.services.bigquery.model.TableRow;
 import com.google.common.base.Function;
 
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -90,6 +91,7 @@ public class BigQueryInterpreter extends Interpreter {
   static final String WAIT_TIME = "zeppelin.bigquery.wait_time";
   static final String MAX_ROWS = "zeppelin.bigquery.max_no_of_rows";
   static final String SQL_DIALECT = "zeppelin.bigquery.sql_dialect";
+  static final String REGION = "zeppelin.bigquery.region";
 
   private static String jobId = null;
   private static String projectId = null;
@@ -227,6 +229,7 @@ public class BigQueryInterpreter extends Interpreter {
     long wTime = Long.parseLong(getProperty(WAIT_TIME));
     long maxRows = Long.parseLong(getProperty(MAX_ROWS));
     String sqlDialect = getProperty(SQL_DIALECT, "").toLowerCase();
+    String region = getProperty(REGION, null);
     Boolean useLegacySql;
     switch (sqlDialect) {
       case "standardsql":
@@ -241,7 +244,7 @@ public class BigQueryInterpreter extends Interpreter {
     }
     Iterator<GetQueryResultsResponse> pages;
     try {
-      pages = run(sql, projId, wTime, maxRows, useLegacySql);
+      pages = run(sql, projId, wTime, maxRows, useLegacySql, region);
     } catch (IOException ex) {
       LOGGER.error(ex.getMessage());
       return new InterpreterResult(Code.ERROR, ex.getMessage());
@@ -258,8 +261,9 @@ public class BigQueryInterpreter extends Interpreter {
 
   //Function to run the SQL on bigQuery service
   public static Iterator<GetQueryResultsResponse> run(final String queryString,
-      final String projId, final long wTime, final long maxRows, Boolean useLegacySql)
-          throws IOException {
+      final String projId, final long wTime, final long maxRows,
+      Boolean useLegacySql, final String region)
+        throws IOException {
     try {
       LOGGER.info("Use legacy sql: {}", useLegacySql);
       QueryResponse query;
@@ -275,6 +279,9 @@ public class BigQueryInterpreter extends Interpreter {
       GetQueryResults getRequest = service.jobs().getQueryResults(
           projectId,
           jobId);
+      if (StringUtils.isNotBlank(region)) {
+        getRequest = getRequest.setLocation(region);
+      }
       return getPages(getRequest);
     } catch (IOException ex) {
       throw ex;
diff --git a/bigquery/src/main/resources/interpreter-setting.json b/bigquery/src/main/resources/interpreter-setting.json
index 8023bed152..989cc375d9 100644
--- a/bigquery/src/main/resources/interpreter-setting.json
+++ b/bigquery/src/main/resources/interpreter-setting.json
@@ -31,6 +31,13 @@
         "defaultValue": "",
         "description": "BigQuery SQL dialect (standardSQL or legacySQL). If empty, query prefix like '#standardSQL' can be used.",
         "type": "string"
+      },
+      "zeppelin.bigquery.region": {
+        "envName": null,
+        "propertyName": "zeppelin.bigquery.region",
+        "defaultValue": "",
+        "description": "Location of BigQuery dataset. Needed if it is a single-region dataset.",
+        "type": "string"
       }
     },
     "editor": {
diff --git a/docs/interpreter/bigquery.md b/docs/interpreter/bigquery.md
index 1a585437f8..da696a74f2 100644
--- a/docs/interpreter/bigquery.md
+++ b/docs/interpreter/bigquery.md
@@ -53,6 +53,11 @@ limitations under the License.
     <td></td>
     <td>BigQuery SQL dialect (standardSQL or legacySQL). If empty, [query prefix](https://cloud.google.com/bigquery/docs/reference/standard-sql/enabling-standard-sql#sql-prefix) like '#standardSQL' can be used.</td>
   </tr>
+  <tr>
+    <td>zeppelin.bigquery.region</td>
+    <td></td>
+    <td>BigQuery dataset region (Needed for single region dataset)</td>
+  </tr>
 </table>