You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2017/01/24 20:27:41 UTC
[1/4] beam git commit: [BEAM-1258] demote retrying loggings to info
level.
Repository: beam
Updated Branches:
refs/heads/master cb6e0a80c -> b3334879f
[BEAM-1258] demote retrying loggings to info level.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3afdc5c0
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3afdc5c0
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3afdc5c0
Branch: refs/heads/master
Commit: 3afdc5c0ef37e48b1750f70e54cd64f5063da83b
Parents: 2a23e8b
Author: Pei He <pe...@google.com>
Authored: Tue Jan 24 11:09:09 2017 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Jan 24 12:25:22 2017 -0800
----------------------------------------------------------------------
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/3afdc5c0/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index c9edf7c..c524ce4 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -220,11 +220,11 @@ class BigQueryServicesImpl implements BigQueryServices {
return; // SUCCEEDED
}
// ignore and retry
- LOG.warn("Ignore the error and retry inserting the job.", e);
+ LOG.info("Ignore the error and retry inserting the job.", e);
lastException = e;
} catch (IOException e) {
// ignore and retry
- LOG.warn("Ignore the error and retry inserting the job.", e);
+ LOG.info("Ignore the error and retry inserting the job.", e);
lastException = e;
}
} while (nextBackOff(sleeper, backoff));
@@ -261,7 +261,7 @@ class BigQueryServicesImpl implements BigQueryServices {
// The job is not DONE, wait longer and retry.
} catch (IOException e) {
// ignore and retry
- LOG.warn("Ignore the error and retry polling job status.", e);
+ LOG.info("Ignore the error and retry polling job status.", e);
}
} while (nextBackOff(sleeper, backoff));
LOG.warn("Unable to poll job status: {}, aborting after reached max .", jobRef.getJobId());
@@ -316,12 +316,12 @@ class BigQueryServicesImpl implements BigQueryServices {
LOG.info("No BigQuery job with job id {} found.", jobId);
return null;
}
- LOG.warn(
+ LOG.info(
"Ignoring the error encountered while trying to query the BigQuery job {}",
jobId, e);
lastException = e;
} catch (IOException e) {
- LOG.warn(
+ LOG.info(
"Ignoring the error encountered while trying to query the BigQuery job {}",
jobId, e);
lastException = e;
@@ -618,10 +618,10 @@ class BigQueryServicesImpl implements BigQueryServices {
return; // SUCCEEDED
}
// ignore and retry
- LOG.warn("Ignore the error and retry creating the dataset.", e);
+ LOG.info("Ignore the error and retry creating the dataset.", e);
lastException = e;
} catch (IOException e) {
- LOG.warn("Ignore the error and retry creating the dataset.", e);
+ LOG.info("Ignore the error and retry creating the dataset.", e);
lastException = e;
}
} while (nextBackOff(sleeper, backoff));
@@ -891,7 +891,7 @@ class BigQueryServicesImpl implements BigQueryServices {
if (!shouldRetry.apply(e)) {
break;
}
- LOG.warn("Ignore the error and retry the request.", e);
+ LOG.info("Ignore the error and retry the request.", e);
}
} while (nextBackOff(sleeper, backoff));
throw new IOException(
[2/4] beam git commit: [BEAM-1302] BigQueryServicesImpl: skip logging
warning if exceptions do not need retry.
Posted by dh...@apache.org.
[BEAM-1302] BigQueryServicesImpl: skip logging warning if exceptions do not need retry.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2a23e8b5
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2a23e8b5
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2a23e8b5
Branch: refs/heads/master
Commit: 2a23e8b5dd11c825dfe13f79d69c2099069be724
Parents: 5b6dd91
Author: Pei He <pe...@google.com>
Authored: Mon Jan 23 17:39:20 2017 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Jan 24 12:25:22 2017 -0800
----------------------------------------------------------------------
.../org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2a23e8b5/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
index 2098148..c9edf7c 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServicesImpl.java
@@ -887,11 +887,11 @@ class BigQueryServicesImpl implements BigQueryServices {
try {
return request.execute();
} catch (IOException e) {
- LOG.warn("Ignore the error and retry the request.", e);
lastException = e;
if (!shouldRetry.apply(e)) {
break;
}
+ LOG.warn("Ignore the error and retry the request.", e);
}
} while (nextBackOff(sleeper, backoff));
throw new IOException(
[3/4] beam git commit: [BEAM-1258] Improve logging in
BigQueryIO.verifyTableEmpty().
Posted by dh...@apache.org.
[BEAM-1258] Improve logging in BigQueryIO.verifyTableEmpty().
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5b6dd91d
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5b6dd91d
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5b6dd91d
Branch: refs/heads/master
Commit: 5b6dd91d27ce73fa66db4d445b0ceb88f09971d8
Parents: cb6e0a8
Author: Pei He <pe...@google.com>
Authored: Mon Jan 23 14:52:30 2017 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Jan 24 12:25:22 2017 -0800
----------------------------------------------------------------------
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 32 +++++++++++---------
.../sdk/io/gcp/bigquery/BigQueryServices.java | 2 ++
2 files changed, 19 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/5b6dd91d/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
index 701374d..aff199a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java
@@ -1863,25 +1863,27 @@ public class BigQueryIO {
writeDisposition, validate, testServices);
}
- private static void verifyTableEmpty(
+ private static void verifyTableNotExistOrEmpty(
DatasetService datasetService,
- TableReference table) {
+ TableReference tableRef) {
try {
- boolean isEmpty = datasetService.isTableEmpty(
- table.getProjectId(), table.getDatasetId(), table.getTableId());
- if (!isEmpty) {
- throw new IllegalArgumentException(
- "BigQuery table is not empty: " + BigQueryIO.toTableSpec(table));
+ if (datasetService.getTable(
+ tableRef.getProjectId(),
+ tableRef.getDatasetId(),
+ tableRef.getTableId()) != null) {
+ checkState(
+ datasetService.isTableEmpty(
+ tableRef.getProjectId(), tableRef.getDatasetId(), tableRef.getTableId()),
+ "BigQuery table is not empty: %s.",
+ BigQueryIO.toTableSpec(tableRef));
}
} catch (IOException | InterruptedException e) {
- ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
- if (e instanceof IOException && errorExtractor.itemNotFound((IOException) e)) {
- // Nothing to do. If the table does not exist, it is considered empty.
- } else {
- throw new RuntimeException(
- "unable to confirm BigQuery table emptiness for table "
- + BigQueryIO.toTableSpec(table), e);
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
}
+ throw new RuntimeException(
+ "unable to confirm BigQuery table emptiness for table "
+ + BigQueryIO.toTableSpec(tableRef), e);
}
}
@@ -1917,7 +1919,7 @@ public class BigQueryIO {
verifyTablePresence(datasetService, table);
}
if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
- verifyTableEmpty(datasetService, table);
+ verifyTableNotExistOrEmpty(datasetService, table);
}
}
http://git-wip-us.apache.org/repos/asf/beam/blob/5b6dd91d/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
----------------------------------------------------------------------
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
index 7173996..32cf46d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryServices.java
@@ -136,6 +136,8 @@ interface BigQueryServices extends Serializable {
/**
* Returns true if the table is empty.
+ *
+ * @throws IOException if the table is not found.
*/
boolean isTableEmpty(String projectId, String datasetId, String tableId)
throws IOException, InterruptedException;
[4/4] beam git commit: This closes #1825
Posted by dh...@apache.org.
This closes #1825
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b3334879
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b3334879
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b3334879
Branch: refs/heads/master
Commit: b3334879fb75150b6f07c24a138fb1d92e1d7def
Parents: cb6e0a8 3afdc5c
Author: Dan Halperin <dh...@google.com>
Authored: Tue Jan 24 12:25:28 2017 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Tue Jan 24 12:25:28 2017 -0800
----------------------------------------------------------------------
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 32 +++++++++++---------
.../sdk/io/gcp/bigquery/BigQueryServices.java | 2 ++
.../io/gcp/bigquery/BigQueryServicesImpl.java | 16 +++++-----
3 files changed, 27 insertions(+), 23 deletions(-)
----------------------------------------------------------------------