You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2017/07/27 23:49:36 UTC
[1/2] incubator-gobblin git commit: Add `CLUSTER` to the
launcher_type enum in the JobHistory DB
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 8d297de3e -> 0975312c7
Add `CLUSTER` to the launcher_type enum in the JobHistory DB
Modified DatabaseJobHistoryStoreV101 to return values of LauncherTypeEnum.CLUSTER as "YARN" and created DatabaseJobHistoryStoreV102 which keeps them as "CLUSTER".
Have DatabaseJobHistoryStoreV102 implement VersionedDatabaseJobHistoryStore
Add missing import
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/6e6883ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/6e6883ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/6e6883ad
Branch: refs/heads/master
Commit: 6e6883adf3240468f3feccb34e9f644d729d7548
Parents: 16dad89
Author: Joel Baranick <jo...@ensighten.com>
Authored: Mon May 1 16:07:17 2017 -0700
Committer: Joel Baranick <jo...@ensighten.com>
Committed: Thu Jun 22 14:32:49 2017 -0700
----------------------------------------------------------------------
.../database/DatabaseJobHistoryStoreV101.java | 12 ++++-
.../database/DatabaseJobHistoryStoreV102.java | 46 ++++++++++++++++++++
.../V1_0_2__LauncherType_enum_expansion.sql | 18 ++++++++
3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
index 6909afc..beeee3f 100644
--- a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
+++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
@@ -250,6 +250,16 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS
return this.dataSource.getConnection();
}
+ protected String getLauncherType(JobExecutionInfo info) {
+ if (info.hasLauncherType()) {
+ if (info.getLauncherType() == LauncherTypeEnum.CLUSTER) {
+ return LauncherTypeEnum.YARN.name();
+ }
+ return info.getLauncherType().name();
+ }
+ return null;
+ }
+
private void upsertJobExecutionInfo(Connection connection, JobExecutionInfo info)
throws SQLException {
Preconditions.checkArgument(info.hasJobName());
@@ -267,7 +277,7 @@ public class DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryS
upsertStatement.setString(++index, info.hasState() ? info.getState().name() : null);
upsertStatement.setInt(++index, info.hasLaunchedTasks() ? info.getLaunchedTasks() : -1);
upsertStatement.setInt(++index, info.hasCompletedTasks() ? info.getCompletedTasks() : -1);
- upsertStatement.setString(++index, info.hasLauncherType() ? info.getLauncherType().name() : null);
+ upsertStatement.setString(++index, getLauncherType(info));
upsertStatement.setString(++index, info.hasTrackingUrl() ? info.getTrackingUrl() : null);
upsertStatement.executeUpdate();
}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
new file mode 100644
index 0000000..f59bf78
--- /dev/null
+++ b/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV102.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package gobblin.metastore.database;
+
+import gobblin.metastore.JobHistoryStore;
+
+import java.io.IOException;
+
+import gobblin.rest.JobExecutionInfo;
+
+
+/**
+ * An implementation of {@link JobHistoryStore} backed by MySQL.
+ *
+ * <p>
+ * The DDLs for the MySQL job history store can be found under metastore/src/main/resources.
+ * </p>
+ *
+ * @author Joel Baranick
+ */
+@SupportedDatabaseVersion(isDefault = false, version = "1.0.2")
+public class DatabaseJobHistoryStoreV102 extends DatabaseJobHistoryStoreV101 implements VersionedDatabaseJobHistoryStore {
+
+ @Override
+ protected String getLauncherType(JobExecutionInfo info) {
+ if (info.hasLauncherType()) {
+ return info.getLauncherType().name();
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/6e6883ad/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql b/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
new file mode 100644
index 0000000..af23b03
--- /dev/null
+++ b/gobblin-metastore/src/main/resources/db/migration/V1_0_2__LauncherType_enum_expansion.sql
@@ -0,0 +1,18 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+ALTER TABLE `gobblin_job_executions` CHANGE `launcher_type` `launcher_type` ENUM('LOCAL', 'MAPREDUCE', 'CLUSTER', 'YARN');
\ No newline at end of file
[2/2] incubator-gobblin git commit: Merge pull request #1966 from
kadaan/Fix_for_#1822_and_#1823
Posted by ab...@apache.org.
Merge pull request #1966 from kadaan/Fix_for_#1822_and_#1823
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/0975312c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/0975312c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/0975312c
Branch: refs/heads/master
Commit: 0975312c7794037ede78ec304252caf706b16a93
Parents: 8d297de 6e6883a
Author: Abhishek Tiwari <ab...@gmail.com>
Authored: Thu Jul 27 16:49:31 2017 -0700
Committer: Abhishek Tiwari <ab...@gmail.com>
Committed: Thu Jul 27 16:49:31 2017 -0700
----------------------------------------------------------------------
.../database/DatabaseJobHistoryStoreV101.java | 12 ++++-
.../database/DatabaseJobHistoryStoreV102.java | 46 ++++++++++++++++++++
.../V1_0_2__LauncherType_enum_expansion.sql | 18 ++++++++
3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/0975312c/gobblin-metastore/src/main/java/gobblin/metastore/database/DatabaseJobHistoryStoreV101.java
----------------------------------------------------------------------