You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by ab...@apache.org on 2018/03/21 08:30:48 UTC
[30/50] incubator-gobblin git commit: [GOBBLIN-409] Set collation to
latin1_bin for the MySql state store backing table
[GOBBLIN-409] Set collation to latin1_bin for the MySql state store backing table
Closes #2286 from
htran1/mysql_state_store_collation
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/c43cf268
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/c43cf268
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/c43cf268
Branch: refs/heads/0.12.0
Commit: c43cf268940bed4d98c0ccba6389d15bacdfaa26
Parents: 0791ee9
Author: Hung Tran <hu...@linkedin.com>
Authored: Tue Feb 13 15:48:12 2018 -0800
Committer: Hung Tran <hu...@linkedin.com>
Committed: Tue Feb 13 15:48:12 2018 -0800
----------------------------------------------------------------------
.../gobblin/metastore/MysqlStateStore.java | 4 +--
.../runtime/MysqlDatasetStateStoreTest.java | 38 +++++++++++++++++++-
2 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c43cf268/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
----------------------------------------------------------------------
diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
index 9756507..b276702 100644
--- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
+++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/MysqlStateStore.java
@@ -101,8 +101,8 @@ public class MysqlStateStore<T extends State> implements StateStore<T> {
// MySQL key length limit is 767 bytes
private static final String CREATE_JOB_STATE_TABLE_TEMPLATE =
- "CREATE TABLE IF NOT EXISTS $TABLE$ (store_name varchar(100) CHARACTER SET latin1 not null,"
- + "table_name varchar(667) CHARACTER SET latin1 not null,"
+ "CREATE TABLE IF NOT EXISTS $TABLE$ (store_name varchar(100) CHARACTER SET latin1 COLLATE latin1_bin not null,"
+ + "table_name varchar(667) CHARACTER SET latin1 COLLATE latin1_bin not null,"
+ " modified_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
+ " state longblob, primary key(store_name, table_name))";
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/c43cf268/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
----------------------------------------------------------------------
diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
index 86ba8ba..45bb44d 100644
--- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
+++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/MysqlDatasetStateStoreTest.java
@@ -43,9 +43,11 @@ public class MysqlDatasetStateStoreTest {
private static final String TEST_STATE_STORE = "TestStateStore";
private static final String TEST_JOB_NAME = "TestJob";
+ private static final String TEST_JOB_NAME_LOWER = "testjob";
private static final String TEST_JOB_ID = "TestJob1";
private static final String TEST_TASK_ID_PREFIX = "TestTask-";
private static final String TEST_DATASET_URN = "TestDataset";
+ private static final String TEST_DATASET_URN_LOWER = "testdataset";
private static final String TEST_DATASET_URN2 = "TestDataset2";
private StateStore<JobState> dbJobStateStore;
@@ -109,6 +111,13 @@ public class MysqlDatasetStateStoreTest {
dbJobStateStore.put(TEST_JOB_NAME,
MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
jobState);
+
+ // for testing a colliding lowercase job name
+ jobState.setJobName(TEST_JOB_NAME_LOWER);
+ jobState.setProp("lower", "case");
+ dbJobStateStore.put(TEST_JOB_NAME_LOWER,
+ MysqlDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + MysqlDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
+ jobState);
}
@Test(dependsOnMethods = "testPersistJobState")
@@ -119,6 +128,8 @@ public class MysqlDatasetStateStoreTest {
Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME);
Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID);
+ Assert.assertEquals(jobState.getProp("foo"), "bar");
+ Assert.assertNotEquals(jobState.getProp("lower"), "case");
Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
Assert.assertEquals(jobState.getStartTime(), this.startTime);
Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000);
@@ -132,6 +143,15 @@ public class MysqlDatasetStateStoreTest {
Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i);
Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
}
+
+ jobState = dbJobStateStore.get(TEST_JOB_NAME_LOWER,
+ dbDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX + dbDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
+ TEST_JOB_ID);
+
+ Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME_LOWER);
+ Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID);
+ Assert.assertEquals(jobState.getProp("foo"), "bar");
+ Assert.assertEquals(jobState.getProp("lower"), "case");
}
@Test(dependsOnMethods = "testGetJobState")
@@ -162,6 +182,13 @@ public class MysqlDatasetStateStoreTest {
datasetState.setDuration(2000);
dbDatasetStateStore.persistDatasetState(TEST_DATASET_URN2, datasetState);
+
+ // persist a colliding lowercase dataset state to test that retrieval is case sensitive
+ datasetState.setDatasetUrn(TEST_DATASET_URN_LOWER);
+ datasetState.setId(TEST_DATASET_URN_LOWER );
+ datasetState.setDuration(3000);
+
+ dbDatasetStateStore.persistDatasetState(TEST_DATASET_URN_LOWER, datasetState);
}
@Test(dependsOnMethods = "testPersistDatasetState")
@@ -191,7 +218,7 @@ public class MysqlDatasetStateStoreTest {
public void testGetPreviousDatasetStatesByUrns() throws IOException {
Map<String, JobState.DatasetState> datasetStatesByUrns =
dbDatasetStateStore.getLatestDatasetStatesByUrns(TEST_JOB_NAME);
- Assert.assertEquals(datasetStatesByUrns.size(), 2);
+ Assert.assertEquals(datasetStatesByUrns.size(), 3);
JobState.DatasetState datasetState = datasetStatesByUrns.get(TEST_DATASET_URN);
Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN);
@@ -210,6 +237,15 @@ public class MysqlDatasetStateStoreTest {
Assert.assertEquals(datasetState.getStartTime(), this.startTime);
Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000);
Assert.assertEquals(datasetState.getDuration(), 2000);
+
+ datasetState = datasetStatesByUrns.get(TEST_DATASET_URN_LOWER);
+ Assert.assertEquals(datasetState.getDatasetUrn(), TEST_DATASET_URN_LOWER);
+ Assert.assertEquals(datasetState.getJobName(), TEST_JOB_NAME);
+ Assert.assertEquals(datasetState.getJobId(), TEST_JOB_ID);
+ Assert.assertEquals(datasetState.getState(), JobState.RunningState.COMMITTED);
+ Assert.assertEquals(datasetState.getStartTime(), this.startTime);
+ Assert.assertEquals(datasetState.getEndTime(), this.startTime + 1000);
+ Assert.assertEquals(datasetState.getDuration(), 3000);
}
@Test(dependsOnMethods = "testGetPreviousDatasetStatesByUrns")