You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pd...@apache.org on 2024/09/09 16:25:43 UTC
(zeppelin) branch master updated: [ZEPPELIN-6080] Add support for bq single region dataset query (#4815)
This is an automated email from the ASF dual-hosted git repository.
pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 205014f0e4 [ZEPPELIN-6080] Add support for bq single region dataset query (#4815)
205014f0e4 is described below
commit 205014f0e4f4abda7a833627c246dd522fb41756
Author: meharanjan318 <45...@users.noreply.github.com>
AuthorDate: Mon Sep 9 21:55:37 2024 +0530
[ZEPPELIN-6080] Add support for bq single region dataset query (#4815)
* Add support for bq single region dataset query
* Remove duplicate code
* Resolve review comments
* Updated documentation
* Improved region check for bq interpreter
---
bigquery/pom.xml | 4 ++++
.../org/apache/zeppelin/bigquery/BigQueryInterpreter.java | 13 ++++++++++---
bigquery/src/main/resources/interpreter-setting.json | 7 +++++++
docs/interpreter/bigquery.md | 5 +++++
4 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/bigquery/pom.xml b/bigquery/pom.xml
index 3a0c8ab64a..a101374d87 100644
--- a/bigquery/pom.xml
+++ b/bigquery/pom.xml
@@ -76,6 +76,10 @@
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </dependency>
</dependencies>
diff --git a/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java b/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
index f80a933b6e..c23bd228e7 100644
--- a/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
+++ b/bigquery/src/main/java/org/apache/zeppelin/bigquery/BigQueryInterpreter.java
@@ -37,6 +37,7 @@ import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.common.base.Function;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,6 +91,7 @@ public class BigQueryInterpreter extends Interpreter {
static final String WAIT_TIME = "zeppelin.bigquery.wait_time";
static final String MAX_ROWS = "zeppelin.bigquery.max_no_of_rows";
static final String SQL_DIALECT = "zeppelin.bigquery.sql_dialect";
+ static final String REGION = "zeppelin.bigquery.region";
private static String jobId = null;
private static String projectId = null;
@@ -227,6 +229,7 @@ public class BigQueryInterpreter extends Interpreter {
long wTime = Long.parseLong(getProperty(WAIT_TIME));
long maxRows = Long.parseLong(getProperty(MAX_ROWS));
String sqlDialect = getProperty(SQL_DIALECT, "").toLowerCase();
+ String region = getProperty(REGION, null);
Boolean useLegacySql;
switch (sqlDialect) {
case "standardsql":
@@ -241,7 +244,7 @@ public class BigQueryInterpreter extends Interpreter {
}
Iterator<GetQueryResultsResponse> pages;
try {
- pages = run(sql, projId, wTime, maxRows, useLegacySql);
+ pages = run(sql, projId, wTime, maxRows, useLegacySql, region);
} catch (IOException ex) {
LOGGER.error(ex.getMessage());
return new InterpreterResult(Code.ERROR, ex.getMessage());
@@ -258,8 +261,9 @@ public class BigQueryInterpreter extends Interpreter {
//Function to run the SQL on bigQuery service
public static Iterator<GetQueryResultsResponse> run(final String queryString,
- final String projId, final long wTime, final long maxRows, Boolean useLegacySql)
- throws IOException {
+ final String projId, final long wTime, final long maxRows,
+ Boolean useLegacySql, final String region)
+ throws IOException {
try {
LOGGER.info("Use legacy sql: {}", useLegacySql);
QueryResponse query;
@@ -275,6 +279,9 @@ public class BigQueryInterpreter extends Interpreter {
GetQueryResults getRequest = service.jobs().getQueryResults(
projectId,
jobId);
+ if (StringUtils.isNotBlank(region)) {
+ getRequest = getRequest.setLocation(region);
+ }
return getPages(getRequest);
} catch (IOException ex) {
throw ex;
diff --git a/bigquery/src/main/resources/interpreter-setting.json b/bigquery/src/main/resources/interpreter-setting.json
index 8023bed152..989cc375d9 100644
--- a/bigquery/src/main/resources/interpreter-setting.json
+++ b/bigquery/src/main/resources/interpreter-setting.json
@@ -31,6 +31,13 @@
"defaultValue": "",
"description": "BigQuery SQL dialect (standardSQL or legacySQL). If empty, query prefix like '#standardSQL' can be used.",
"type": "string"
+ },
+ "zeppelin.bigquery.region": {
+ "envName": null,
+ "propertyName": "zeppelin.bigquery.region",
+ "defaultValue": "",
+ "description": "Location of BigQuery dataset. Needed if it is a single-region dataset.",
+ "type": "string"
}
},
"editor": {
diff --git a/docs/interpreter/bigquery.md b/docs/interpreter/bigquery.md
index 1a585437f8..da696a74f2 100644
--- a/docs/interpreter/bigquery.md
+++ b/docs/interpreter/bigquery.md
@@ -53,6 +53,11 @@ limitations under the License.
<td></td>
<td>BigQuery SQL dialect (standardSQL or legacySQL). If empty, [query prefix](https://cloud.google.com/bigquery/docs/reference/standard-sql/enabling-standard-sql#sql-prefix) like '#standardSQL' can be used.</td>
</tr>
+ <tr>
+ <td>zeppelin.bigquery.region</td>
+ <td></td>
+ <td>BigQuery dataset region (Needed for single region dataset)</td>
+ </tr>
</table>