You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@omid.apache.org by ik...@apache.org on 2016/04/21 02:02:18 UTC
[01/52] [abbrv] incubator-omid git commit: Updated source code with
license header
Repository: incubator-omid
Updated Branches:
refs/heads/master 27964b210 -> e5b4ca315
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
index c99e174..44a0cb3 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import org.apache.curator.framework.CuratorFramework;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java b/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
index e0ca51b..d8d1d2c 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java b/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
index 1c9db56..b882779 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.google.inject.AbstractModule;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java b/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
index 5a596a8..7872589 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
index 54b87b0..74879b0 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.yahoo.omid.metrics.MetricsRegistry;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
index cf099f5..4da3c45 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.google.common.base.Charsets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
index b75c006..6e54302 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
index 1e71783..c9bd743 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.yahoo.omid.committable.CommitTable;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
index 7b28d43..03d4ffc 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.yahoo.omid.committable.CommitTable;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
index ab4b7c3..d3e2c18 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
index 6407b68..decd75e 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
index 6f84acf..3f071c0 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
index 3c523bb..eda0985 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.yahoo.omid.tso.TSOStateManager.StateObserver;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
index cbd7bcd..3a6f1b9 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tso;
import com.yahoo.omid.metrics.MetricsRegistry;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
index 233800e..d98b353 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.yahoo.omid.tsoclient.TSOClient.ConnectedState;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
index b223ebb..eb0a49c 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.yahoo.omid.proto.TSOProto;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
index 4244a3c..e6f7ebe 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.google.common.util.concurrent.SettableFuture;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
index b89ab0c..5185d83 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.google.common.collect.Sets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
index ede61c8..d41517c 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.google.inject.Guice;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
index 088e281..33b12a9 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.google.common.collect.Sets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
index 8aa6771..66c3628 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.yahoo.omid.tso.ProgrammableTSOServer;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
index 4ce1077..641ab0a 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.yahoo.omid.tso.util.DummyCellIdImpl;
[47/52] [abbrv] incubator-omid git commit: Merge pull request #91
from yahoo/website-generation
Posted by ik...@apache.org.
Merge pull request #91 from yahoo/website-generation
Add infrastructure to create Omid website
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/d2a26c48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/d2a26c48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/d2a26c48
Branch: refs/heads/master
Commit: d2a26c4895bb9544d4528958629dc1587b8aa3ce
Parents: fd8ca18 05d69d9
Author: ikatkov <ik...@gmail.com>
Authored: Tue Apr 19 14:11:10 2016 -0700
Committer: ikatkov <ik...@gmail.com>
Committed: Tue Apr 19 14:11:10 2016 -0700
----------------------------------------------------------------------
.gitignore | 8 +-
benchmarks/pom.xml | 9 +
codahale-metrics/pom.xml | 16 ++
commit-table/pom.xml | 8 +
common/pom.xml | 8 +
doc/site/markdown/basic-algorithm.md | 29 ++
doc/site/markdown/basic-concepts.md | 79 ++++++
doc/site/markdown/basic-examples.md | 264 +++++++++++++++++++
doc/site/markdown/client-failure-management.md | 30 +++
doc/site/markdown/coding-guide-and-style.md | 56 ++++
doc/site/markdown/index.md | 94 +++++++
doc/site/markdown/mailing-lists.md | 13 +
doc/site/markdown/omid-components.md | 96 +++++++
doc/site/markdown/quickstart.md | 203 ++++++++++++++
doc/site/resources/css/site.css | 103 ++++++++
doc/site/resources/images/about-omid.png | Bin 0 -> 3402 bytes
doc/site/resources/images/architecture.png | Bin 0 -> 77416 bytes
doc/site/resources/images/basic-alg.png | Bin 0 -> 30454 bytes
doc/site/resources/images/contact.png | Bin 0 -> 2899 bytes
doc/site/resources/images/getting-started.png | Bin 0 -> 4328 bytes
.../resources/images/omid-logo-transparent.png | Bin 0 -> 13867 bytes
doc/site/resources/images/omid-logo.png | Bin 0 -> 23473 bytes
.../resources/images/snapshot-isolation.png | Bin 0 -> 22535 bytes
.../images/technical-documentation.png | Bin 0 -> 2857 bytes
doc/site/site.xml | 85 ++++++
examples/pom.xml | 8 +
hbase-client/pom.xml | 7 +
hbase-commit-table/pom.xml | 16 ++
hbase-common/pom.xml | 16 ++
hbase-coprocessor/pom.xml | 8 +
hbase-shims/pom.xml | 16 ++
hbase-tools/pom.xml | 16 ++
metrics/pom.xml | 16 ++
pom.xml | 111 +++++++-
statemachine/pom.xml | 16 ++
timestamp-storage/pom.xml | 16 ++
transaction-client/pom.xml | 16 ++
tso-server/pom.xml | 8 +
38 files changed, 1362 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[19/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TSOServer.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TSOServer.java b/tso-server/src/main/java/org/apache/omid/tso/TSOServer.java
new file mode 100644
index 0000000..b838666
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TSOServer.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.google.inject.Singleton;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.tools.hbase.HBaseLogin;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@Singleton
+public class TSOServer extends AbstractIdleService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOServer.class);
+
+ public static final String TSO_HOST_AND_PORT_KEY = "tso.hostandport";
+
+ @Inject
+ private TSOStateManager tsoStateManager;
+ @Inject
+ private RequestProcessor requestProcessor;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // High availability related variables
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Inject
+ private LeaseManagement leaseManagement;
+
+ // ----------------------------------------------------------------------------------------------------------------
+
+ static TSOServer getInitializedTsoServer(TSOServerConfig config) throws IOException {
+ LOG.info("Configuring TSO Server...");
+ Injector injector = Guice.createInjector(buildModuleList(config));
+ LOG.info("TSO Server configured. Creating instance...");
+ return injector.getInstance(TSOServer.class);
+ }
+
+ private static List<Module> buildModuleList(final TSOServerConfig config) throws IOException {
+
+ List<Module> guiceModules = new ArrayList<>();
+ guiceModules.add(config.getTimestampStoreModule());
+ guiceModules.add(config.getCommitTableStoreModule());
+ guiceModules.add(config.getLeaseModule());
+ guiceModules.add(new TSOModule(config));
+
+ guiceModules.add(new Module() {
+ @Override
+ public void configure(Binder binder) {
+ LOG.info("\t* Metrics provider module set to {}", config.getMetrics().getClass());
+ binder.bind(MetricsRegistry.class).toInstance(config.getMetrics());
+ }
+ });
+ return guiceModules;
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // AbstractIdleService implementation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ protected void startUp() throws Exception {
+ tsoStateManager.register(requestProcessor);
+ leaseManagement.startService();
+ LOG.info("********** TSO Server running **********");
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ leaseManagement.stopService();
+ tsoStateManager.unregister(requestProcessor);
+ LOG.info("********** TSO Server stopped successfully **********");
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private void attachShutDownHook() {
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ stopAndWait();
+ }
+ });
+ LOG.info("Shutdown Hook Attached");
+ }
+
+ /**
+ * This is where all starts on the server side
+ */
+ public static void main(String[] args) {
+
+ TSOServerConfig config = new TSOServerConfig();
+
+ try {
+ TSOServer tsoServer = getInitializedTsoServer(config);
+ tsoServer.attachShutDownHook();
+ tsoServer.startAndWait();
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ System.exit(-1);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TSOServerConfig.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TSOServerConfig.java b/tso-server/src/main/java/org/apache/omid/tso/TSOServerConfig.java
new file mode 100644
index 0000000..9541eec
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TSOServerConfig.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Module;
+import org.apache.omid.YAMLUtils;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.util.Enumeration;
+
+/**
+ * Reads the configuration parameters of a TSO server instance from CONFIG_FILE_NAME.
+ * If file CONFIG_FILE_NAME is missing defaults to DEFAULT_CONFIG_FILE_NAME
+ */
+@SuppressWarnings("all")
+public class TSOServerConfig extends SecureHBaseConfig {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOServerConfig.class);
+
+ private static final String DEFAULT_CONFIG_FILE_NAME = "default-omid.yml";
+ private static final String CONFIG_FILE_NAME = "omid.yml";
+
+ private static final String LINUX_TSO_NET_IFACE_PREFIX = "eth";
+ private static final String MAC_TSO_NET_IFACE_PREFIX = "en";
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Instantiation
+ // ----------------------------------------------------------------------------------------------------------------
+ public TSOServerConfig() {
+ this(CONFIG_FILE_NAME);
+ }
+
+ @VisibleForTesting
+ TSOServerConfig(String configFileName) {
+ new YAMLUtils().loadSettings(configFileName, DEFAULT_CONFIG_FILE_NAME, this);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Configuration parameters
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private Module timestampStoreModule;
+
+ private Module commitTableStoreModule;
+
+ private Module leaseModule;
+
+ private int port;
+
+ private MetricsRegistry metrics;
+
+ private int maxItems;
+
+ private int maxBatchSize;
+
+ private int batchPersistTimeoutInMs;
+
+ private String networkIfaceName = getDefaultNetworkInterface();
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ public void setMaxBatchSize(int maxBatchSize) {
+ this.maxBatchSize = maxBatchSize;
+ }
+
+ public void setBatchPersistTimeoutInMs(int value) {
+ this.batchPersistTimeoutInMs = value;
+ }
+
+ public String getNetworkIfaceName() {
+ return networkIfaceName;
+ }
+
+ public void setNetworkIfaceName(String networkIfaceName) {
+ this.networkIfaceName = networkIfaceName;
+ }
+
+ public Module getTimestampStoreModule() {
+ return timestampStoreModule;
+ }
+
+ public void setTimestampStoreModule(Module timestampStoreModule) {
+ this.timestampStoreModule = timestampStoreModule;
+ }
+
+ public Module getCommitTableStoreModule() {
+ return commitTableStoreModule;
+ }
+
+ public void setCommitTableStoreModule(Module commitTableStoreModule) {
+ this.commitTableStoreModule = commitTableStoreModule;
+ }
+
+ public Module getLeaseModule() {
+ return leaseModule;
+ }
+
+ public void setLeaseModule(Module leaseModule) {
+ this.leaseModule = leaseModule;
+ }
+
+ public int getMaxItems() {
+ return maxItems;
+ }
+
+ public void setMaxItems(int maxItems) {
+ this.maxItems = maxItems;
+ }
+
+ public int getMaxBatchSize() {
+ return maxBatchSize;
+ }
+
+ public int getBatchPersistTimeoutInMs() {
+ return batchPersistTimeoutInMs;
+ }
+
+ public MetricsRegistry getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(MetricsRegistry metrics) {
+ this.metrics = metrics;
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private String getDefaultNetworkInterface() {
+
+ try {
+ Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
+ while (networkInterfaces.hasMoreElements()) {
+ String name = networkInterfaces.nextElement().getDisplayName();
+ LOG.info("Iterating over network interfaces, found '{}'", name);
+ if (name.startsWith(MAC_TSO_NET_IFACE_PREFIX) || name.startsWith(LINUX_TSO_NET_IFACE_PREFIX)) {
+ return name;
+ }
+ }
+ } catch (SocketException ignored) {
+ throw new RuntimeException("Failed to find any network interfaces", ignored);
+ }
+ throw new IllegalArgumentException(String.format("No network '%s*'/'%s*' interfaces found",
+ MAC_TSO_NET_IFACE_PREFIX, LINUX_TSO_NET_IFACE_PREFIX));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TSOStateManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TSOStateManager.java b/tso-server/src/main/java/org/apache/omid/tso/TSOStateManager.java
new file mode 100644
index 0000000..eeaab9d
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TSOStateManager.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import java.io.IOException;
+
+/**
+ * Allows to reset the TSO state and register observers for being notified
+ * when changes occur
+ */
+public interface TSOStateManager {
+
+ /**
+ * Represents the state of the TSO
+ */
+ public static class TSOState {
+
+ // TSO state variables
+ private final long lowWatermark;
+
+ public TSOState(long lowWatermark, long epoch) {
+ this.lowWatermark = lowWatermark;
+ }
+
+ public long getLowWatermark() {
+ return lowWatermark;
+ }
+
+ public long getEpoch() {
+ // In this implementation the epoch == low watermark
+ return lowWatermark;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("LWM %d/Epoch %d", getLowWatermark(), getEpoch());
+ }
+
+ }
+
+ /**
+ * Allows implementors to receive the new state when changes occur
+ */
+ public interface StateObserver {
+
+ /**
+ * Notifies the observer about the change in state
+ * @param state
+ * the new TSOState
+ */
+ public void update(TSOState state) throws IOException;
+
+ }
+
+ /**
+ * Allows to register observers for receiving state changes
+ *
+ * @param observer
+ * the observer to register
+ */
+ public void register(StateObserver observer);
+
+ /**
+ * Allows to de-register observers for stopping receiving changes
+ *
+ * @param observer
+ * the observer to unregister
+ */
+ public void unregister(StateObserver observer);
+
+ /**
+ * Allows to reset the state
+ *
+ * @return the new state after reset
+ * @throws IOException
+ * when problems resetting occur
+ */
+ public TSOState reset() throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TSOStateManagerImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TSOStateManagerImpl.java b/tso-server/src/main/java/org/apache/omid/tso/TSOStateManagerImpl.java
new file mode 100644
index 0000000..ce5072d
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TSOStateManagerImpl.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Implements the management of the state of the TSO
+ */
+public class TSOStateManagerImpl implements TSOStateManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOStateManagerImpl.class);
+
+ private List<StateObserver> stateObservers = new ArrayList<>();
+
+ private TSOState state;
+
+ private TimestampOracle timestampOracle;
+
+ @Inject
+ public TSOStateManagerImpl(TimestampOracle timestampOracle) {
+ this.timestampOracle = timestampOracle;
+ }
+
+ @Override
+ public synchronized void register(StateObserver newObserver) {
+ Preconditions.checkNotNull(newObserver, "Trying to register a null observer");
+ if (!stateObservers.contains(newObserver)) {
+ stateObservers.add(newObserver);
+ }
+ }
+
+ @Override
+ public synchronized void unregister(StateObserver observer) {
+ stateObservers.remove(observer);
+ }
+
+ @Override
+ public synchronized TSOState reset() throws IOException {
+ LOG.info("Reseting the TSO Server state...");
+ // The timestamp oracle dictates the new state
+ timestampOracle.initialize();
+ long lowWatermark = timestampOracle.getLast();
+ // In this implementation the epoch == low watermark
+ long epoch = lowWatermark;
+ state = new TSOState(lowWatermark, epoch);
+
+ // Then, notify registered observers about the new state
+ for (StateObserver stateObserver : stateObservers) {
+ stateObserver.update(state);
+ }
+ LOG.info("New TSO Server state {}", state);
+ return state;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TimestampOracle.java b/tso-server/src/main/java/org/apache/omid/tso/TimestampOracle.java
new file mode 100644
index 0000000..76c6ab3
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TimestampOracle.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import java.io.IOException;
+
+public interface TimestampOracle {
+
+ void initialize() throws IOException;
+
+ /**
+ * Returns the next timestamp if available. Otherwise spins till the
+ * ts-persist thread performs the new timestamp allocation
+ */
+ long next() throws IOException;
+
+ long getLast();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TimestampOracleImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TimestampOracleImpl.java b/tso-server/src/main/java/org/apache/omid/tso/TimestampOracleImpl.java
new file mode 100644
index 0000000..2c337a1
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TimestampOracleImpl.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.metrics.Gauge;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.io.IOException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import static org.apache.omid.metrics.MetricsUtils.name;
+
+/**
+ * The Timestamp Oracle that gives monotonically increasing timestamps
+ */
+@Singleton
+public class TimestampOracleImpl implements TimestampOracle {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TimestampOracleImpl.class);
+
+ @VisibleForTesting
+ static class InMemoryTimestampStorage implements TimestampStorage {
+
+ long maxTimestamp = 0;
+
+ @Override
+ public void updateMaxTimestamp(long previousMaxTimestamp, long nextMaxTimestamp) {
+ maxTimestamp = nextMaxTimestamp;
+ LOG.info("Updating max timestamp: (previous:{}, new:{})", previousMaxTimestamp, nextMaxTimestamp);
+ }
+
+ @Override
+ public long getMaxTimestamp() {
+ return maxTimestamp;
+ }
+
+ }
+
+ private class AllocateTimestampBatchTask implements Runnable {
+ long previousMaxTimestamp;
+
+ AllocateTimestampBatchTask(long previousMaxTimestamp) {
+ this.previousMaxTimestamp = previousMaxTimestamp;
+ }
+
+ @Override
+ public void run() {
+ long newMaxTimestamp = previousMaxTimestamp + TIMESTAMP_BATCH;
+ try {
+ storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
+ maxAllocatedTimestamp = newMaxTimestamp;
+ previousMaxTimestamp = newMaxTimestamp;
+ } catch (Throwable e) {
+ panicker.panic("Can't store the new max timestamp", e);
+ }
+ }
+
+ }
+
+ static final long TIMESTAMP_BATCH = 10_000_000; // 10 million
+ private static final long TIMESTAMP_REMAINING_THRESHOLD = 1_000_000; // 1 million
+
+ private long lastTimestamp;
+
+ private long maxTimestamp;
+
+ private TimestampStorage storage;
+ private Panicker panicker;
+
+ private long nextAllocationThreshold;
+ private volatile long maxAllocatedTimestamp;
+
+ private Executor executor = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("ts-persist-%d").build());
+
+ private Runnable allocateTimestampsBatchTask;
+
+ @Inject
+ public TimestampOracleImpl(MetricsRegistry metrics,
+ TimestampStorage tsStorage,
+ Panicker panicker) throws IOException {
+
+ this.storage = tsStorage;
+ this.panicker = panicker;
+
+ metrics.gauge(name("tso", "maxTimestamp"), new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return maxTimestamp;
+ }
+ });
+
+ }
+
+ @Override
+ public void initialize() throws IOException {
+
+ this.lastTimestamp = this.maxTimestamp = storage.getMaxTimestamp();
+
+ this.allocateTimestampsBatchTask = new AllocateTimestampBatchTask(lastTimestamp);
+
+ // Trigger first allocation of timestamps
+ executor.execute(allocateTimestampsBatchTask);
+
+ LOG.info("Initializing timestamp oracle with timestamp {}", this.lastTimestamp);
+ }
+
+ /**
+ * Returns the next timestamp if available. Otherwise spins till the
+ * ts-persist thread performs the new timestamp allocation
+ */
+ @SuppressWarnings("StatementWithEmptyBody")
+ @Override
+ public long next() throws IOException {
+ lastTimestamp++;
+
+ if (lastTimestamp == nextAllocationThreshold) {
+ executor.execute(allocateTimestampsBatchTask);
+ }
+
+ if (lastTimestamp >= maxTimestamp) {
+ assert (maxTimestamp <= maxAllocatedTimestamp);
+ while (maxAllocatedTimestamp == maxTimestamp) {
+ // spin
+ }
+ assert (maxAllocatedTimestamp > maxTimestamp);
+ maxTimestamp = maxAllocatedTimestamp;
+ nextAllocationThreshold = maxTimestamp - TIMESTAMP_REMAINING_THRESHOLD;
+ assert (nextAllocationThreshold > lastTimestamp && nextAllocationThreshold < maxTimestamp);
+ assert (lastTimestamp < maxTimestamp);
+ }
+
+ return lastTimestamp;
+ }
+
+ @Override
+ public long getLast() {
+ return lastTimestamp;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("TimestampOracle -> LastTimestamp: %d, MaxTimestamp: %d", lastTimestamp, maxTimestamp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TsoServerDaemon.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TsoServerDaemon.java b/tso-server/src/main/java/org/apache/omid/tso/TsoServerDaemon.java
new file mode 100644
index 0000000..fae0e73
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TsoServerDaemon.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.commons.daemon.Daemon;
+import org.apache.commons.daemon.DaemonContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Wraps TSO Server as a Daemon
+ */
+public class TsoServerDaemon implements Daemon {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TsoServerDaemon.class);
+ private TSOServer tsoServer;
+
+ @Override
+ public void init(DaemonContext daemonContext) throws Exception {
+ try {
+ TSOServerConfig config = new TSOServerConfig();
+ LOG.info("Creating TSOServer instance as a Daemon process...");
+ tsoServer = TSOServer.getInitializedTsoServer(config);
+ LOG.info("TSOServer instance for Daemon process created");
+ } catch (Exception e) {
+ LOG.error("Error creating TSOServer instance", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+ tsoServer.startAndWait();
+ }
+
+ @Override
+ public void stop() throws Exception {
+ tsoServer.stopAndWait();
+ }
+
+ @Override
+ public void destroy() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManagementModule.java b/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManagementModule.java
new file mode 100644
index 0000000..6f0c590
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManagementModule.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.name.Names;
+import org.apache.omid.HBaseConfigModule;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
+import org.apache.omid.tso.LeaseManagement.LeaseManagementException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Singleton;
+
+import static org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+
+public class VoidLeaseManagementModule extends AbstractModule {
+ private static final Logger LOG = LoggerFactory.getLogger(VoidLeaseManagementModule.class);
+ private String keytab;
+ private String principal;
+ private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+
+ @Override
+ protected void configure() {
+ LOG.info(this.toString());
+ bindConstant().annotatedWith(Names.named(HBaseTimestampStorageConfig.TIMESTAMP_STORAGE_TABLE_NAME_KEY)).to(tableName);
+ install(new HBaseConfigModule(principal, keytab));
+ }
+
+ @Provides
+ @Singleton
+ LeaseManagement provideLeaseManager(TSOChannelHandler tsoChannelHandler, TSOStateManager stateManager)
+ throws LeaseManagementException {
+
+ return new VoidLeaseManager(tsoChannelHandler, stateManager);
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // WARNING: Do not remove getters/setters, needed by snake_yaml!
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public String getKeytab() {
+ return keytab;
+ }
+
+ public void setKeytab(String keytab) {
+ this.keytab = keytab;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("VoidLeaseManagementModule{");
+ sb.append("keytab='").append(keytab).append('\'');
+ sb.append(", principal='").append(principal).append('\'');
+ sb.append(", tableName='").append(tableName).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManager.java b/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManager.java
new file mode 100644
index 0000000..60373cf
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/VoidLeaseManager.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import java.io.IOException;
+
+public class VoidLeaseManager implements LeaseManagement {
+
+ private final TSOChannelHandler tsoChannelHandler;
+ private TSOStateManager stateManager;
+
+ public VoidLeaseManager(TSOChannelHandler tsoChannelHandler, TSOStateManager stateManager) {
+ this.tsoChannelHandler = tsoChannelHandler;
+ this.stateManager = stateManager;
+ }
+
+ @Override
+ public void startService() throws LeaseManagementException {
+ try {
+ stateManager.reset();
+ tsoChannelHandler.reconnect();
+ } catch (IOException e) {
+ throw new LeaseManagementException("Error initializing Lease Manager", e);
+ }
+ }
+
+ @Override
+ public void stopService() throws LeaseManagementException {
+ tsoChannelHandler.closeConnection();
+ }
+
+ @Override
+ public boolean stillInLeasePeriod() {
+ // We should always return true
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/resources/default-omid.yml
----------------------------------------------------------------------
diff --git a/tso-server/src/main/resources/default-omid.yml b/tso-server/src/main/resources/default-omid.yml
index 5d18443..85b2b53 100644
--- a/tso-server/src/main/resources/default-omid.yml
+++ b/tso-server/src/main/resources/default-omid.yml
@@ -10,7 +10,7 @@
# Network interface for TSO server communication. Uncomment the following line to use a specific interface
# networkIfaceName: eth0
# If a network interface in the configuration, the TSO will attempt to guess default network interface.
-# See com.yahoo.omid.tso.TSOServerConfig.getDefaultNetworkInterface for more information.
+# See org.apache.omid.tso.TSOServerConfig.getDefaultNetworkInterface for more information.
# Port reserved by the Status Oracle
port: 54758
@@ -19,44 +19,44 @@ maxBatchSize: 25
batchPersistTimeoutInMs: 10
# Default module configuration (No TSO High Availability & in-memory storage for timestamp and commit tables)
-timestampStoreModule: !!com.yahoo.omid.tso.InMemoryTimestampStorageModule [ ]
-commitTableStoreModule: !!com.yahoo.omid.tso.InMemoryCommitTableStorageModule [ ]
-leaseModule: !!com.yahoo.omid.tso.VoidLeaseManagementModule [ ]
+timestampStoreModule: !!org.apache.omid.tso.InMemoryTimestampStorageModule [ ]
+commitTableStoreModule: !!org.apache.omid.tso.InMemoryCommitTableStorageModule [ ]
+leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
# Default stats/metrics configuration (
-metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
+metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
# ---------------------------------------------------------------------------------------------------------------------
# Timestamp storage configuration options
# ---------------------------------------------------------------------------------------------------------------------
-# Could be any guava module that binds com.yahoo.omid.timestamp.storage.TimestampStorage
+# Could be any guava module that binds org.apache.omid.timestamp.storage.TimestampStorage
# Current available Timestamp stores:
-# com.yahoo.omid.tso.InMemoryTimestampStorageModule
-# com.yahoo.omid.timestamp.storage.HBaseTimestampStorageModule
-# com.yahoo.omid.timestamp.storage.ZKTimestampStorageModule
+# org.apache.omid.tso.InMemoryTimestampStorageModule
+# org.apache.omid.timestamp.storage.HBaseTimestampStorageModule
+# org.apache.omid.timestamp.storage.ZKTimestampStorageModule
# ---------------------------------------------------------------------------------------------------------------------
# Commit Table storage configuration options
# ---------------------------------------------------------------------------------------------------------------------
-# Could be any guava module that binds com.yahoo.omid.committable.CommitTable
+# Could be any guava module that binds org.apache.omid.committable.CommitTable
# Available CommitTable stores:
-# com.yahoo.omid.committable.hbase.HBaseCommitTableStorageModule
-# com.yahoo.omid.tso.InMemoryCommitTableStorageModule
+# org.apache.omid.committable.hbase.HBaseCommitTableStorageModule
+# org.apache.omid.tso.InMemoryCommitTableStorageModule
# ---------------------------------------------------------------------------------------------------------------------
# Metrics configuration options
# ---------------------------------------------------------------------------------------------------------------------
-# Metrics could be anything that is com.yahoo.omid.metrics.MetricsRegistry
+# Metrics could be anything that is org.apache.omid.metrics.MetricsRegistry
# There are 4 built-in reporters: CSV, SLF4J, GRAPHITE, CONSOLE
-# Please see com.yahoo.omid.metrics.CodahaleMetricsConfig for details.
+# Please see org.apache.omid.metrics.CodahaleMetricsConfig for details.
# Example configuration for reporting statistics to the console every minute:
#
-# metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
# outputFreqInSecs: 60,
# reporters: !!set {
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
# },
# }
# ]
@@ -64,12 +64,12 @@ metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
# Example of multiple reporter configuration (to CSV files and console)
#
-# metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
# outputFreqInSecs: 60,
# reporters: !!set {
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
# },
# csvDir: "csvMetrics",
# prefix: "somePrefix",
@@ -84,23 +84,23 @@ metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
# Configuration WITHOUT High Availability using HBase for all required storage & reporting metrics to CSV files
# ---------------------------------------------------------------------------------------------------------------------
#
-# commitTableStoreModule: !!com.yahoo.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
# See optional params
# - tableName
# - familyName
# - principal
# - keytab
-# timestampStoreModule: !!com.yahoo.omid.tso.DefaultHBaseTimestampStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultHBaseTimestampStorageModule [ ]
# See optional params
# - tableName
# - familyName
# - principal
# - keytab
-# leaseModule: !!com.yahoo.omid.tso.VoidLeaseManagementModule [ ]
-# metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+# leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
# reporters: !!set {
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CSV
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV
# },
# csvDir: "myCSVMetricsDir",
# prefix: "myAppPrefixForMetrics",
@@ -111,16 +111,16 @@ metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
# Configuration WITHOUT High Availability using ZK to store the timestamps & reporting metrics to console every 30 secs
# ---------------------------------------------------------------------------------------------------------------------
#
-# commitTableStoreModule: !!com.yahoo.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
-# timestampStoreModule: !!com.yahoo.omid.tso.DefaultZKTimestampStorageModule
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultZKTimestampStorageModule
# zkCluster: "localhost:2181"
# namespace: "omid"
-# leaseModule: !!com.yahoo.omid.tso.VoidLeaseManagementModule [ ]
-# metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+# leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
+# metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+# !!org.apache.omid.metrics.CodahaleMetricsConfig {
# outputFreqInSecs: 30,
# reporters: !!set {
-# !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+# !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
# },
# }
# ]
@@ -130,16 +130,16 @@ metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
# Configuration WITH High Availability using HBase for all required storage and no metrics reports
# ---------------------------------------------------------------------------------------------------------------------
#
-# commitTableStoreModule: !!com.yahoo.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
-# timestampStoreModule: !!com.yahoo.omid.tso.DefaultHBaseTimestampStorageModule [ ]
-# leaseModule: !!com.yahoo.omid.tso.HALeaseManagementModule [
+# commitTableStoreModule: !!org.apache.omid.tso.DefaultHBaseCommitTableStorageModule [ ]
+# timestampStoreModule: !!org.apache.omid.tso.DefaultHBaseTimestampStorageModule [ ]
+# leaseModule: !!org.apache.omid.tso.HALeaseManagementModule [
# leasePeriodInMs: 10000,
# tsoLeasePath: "/tso-lease",
# zkCurrentTsoPath: "/current-tso",
# zkCluster: "localhost:2181",
# namespace: "omid"
# ]
-# metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
+# metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tso-server/src/main/resources/log4j.properties b/tso-server/src/main/resources/log4j.properties
index 5884e05..1633250 100644
--- a/tso-server/src/main/resources/log4j.properties
+++ b/tso-server/src/main/resources/log4j.properties
@@ -37,13 +37,12 @@ log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.bookkeeper=ERROR
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.apache.hadoop.ipc=ERROR
-
-log4j.logger.com.yahoo.omid=INFO
-#log4j.logger.com.yahoo.omid.regionserver.TransactionalRegionServer=TRACE
-#log4j.logger.com.yahoo.omid.TestBasicTransaction=TRACE
-#log4j.logger.com.yahoo.omid.client.TSOClient=TRACE
-#log4j.logger.com.yahoo.omid.client.TransactionState=TRACE
-#log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=TRACE
+log4j.logger.org.apache.omid=INFO
+#log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE
+#log4j.logger.org.apache.omid.TestBasicTransaction=TRACE
+#log4j.logger.org.apache.omid.client.TSOClient=TRACE
+#log4j.logger.org.apache.omid.client.TransactionState=TRACE
+#log4j.logger.org.apache.omid.tso.ThroughputMonitor=TRACE
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/TestUtils.java b/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
deleted file mode 100644
index 7c30691..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.test.TestingServer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.Socket;
-
-/**
- * This class contains functionality that is useful for the Omid tests.
- */
-public class TestUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
-
- private static final int DEFAULT_ZK_PORT = 2181;
-
- public static int getFreeLocalPort() throws IOException {
-
- try (ServerSocket socket = new ServerSocket(0)) {
- socket.setReuseAddress(true);
- return socket.getLocalPort();
- }
-
- }
-
- public static TestingServer provideTestingZKServer(int port) throws Exception {
-
- return new TestingServer(port);
-
- }
-
- public static TestingServer provideTestingZKServer() throws Exception {
-
- return provideTestingZKServer(DEFAULT_ZK_PORT);
-
- }
-
- public static CuratorFramework provideConnectedZKClient(String zkCluster) throws Exception {
-
- LOG.info("Creating Zookeeper Client connecting to {}", zkCluster);
-
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- CuratorFramework zkClient = CuratorFrameworkFactory.builder().namespace("omid")
- .connectString(zkCluster).retryPolicy(retryPolicy).build();
-
- LOG.info("Connecting to ZK cluster {}", zkClient.getState());
- zkClient.start();
- zkClient.blockUntilConnected();
- LOG.info("Connection to ZK cluster {}", zkClient.getState());
-
- return zkClient;
- }
-
- public static void waitForSocketListening(String host, int port, int sleepTimeMillis)
- throws IOException, InterruptedException {
- while (true) {
- Socket sock = null;
- try {
- sock = new Socket(host, port);
- } catch (IOException e) {
- // ignore as this is expected
- Thread.sleep(sleepTimeMillis);
- continue;
- } finally {
- IOUtils.closeQuietly(sock);
- }
- LOG.info("Host " + host + ":" + port + " is up...");
- break;
- }
- }
-
- public static void waitForSocketNotListening(String host, int port, int sleepTimeMillis)
- throws IOException, InterruptedException {
- while (true) {
- Socket sock = null;
- try {
- sock = new Socket(host, port);
- } catch (IOException e) {
- // ignore as this is expected
- break;
- } finally {
- IOUtils.closeQuietly(sock);
- }
- Thread.sleep(sleepTimeMillis);
- LOG.info("Host " + host + ":" + port + " is still up...");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
deleted file mode 100644
index 44a0cb3..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/PausableLeaseManager.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class PausableLeaseManager extends LeaseManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(PausableLeaseManager.class);
-
- private volatile boolean pausedInTryToGetInitialLeasePeriod = false;
- private volatile boolean pausedInTryToRenewLeasePeriod = false;
- private volatile boolean pausedInStillInLeasePeriod = false;
-
- public PausableLeaseManager(String id,
- TSOChannelHandler tsoChannelHandler,
- TSOStateManager stateManager,
- long leasePeriodInMs,
- String tsoLeasePath,
- String currentTSOPath,
- CuratorFramework zkClient,
- Panicker panicker) {
- super(id, tsoChannelHandler, stateManager, leasePeriodInMs, tsoLeasePath, currentTSOPath, zkClient, panicker);
- }
-
- @Override
- public void tryToGetInitialLeasePeriod() throws Exception {
- while (pausedInTryToGetInitialLeasePeriod) {
- synchronized (this) {
- try {
- LOG.info("{} paused in tryToGetInitialLeasePeriod()", this);
- this.wait();
- } catch (InterruptedException e) {
- LOG.error("Interrupted whilst paused");
- Thread.currentThread().interrupt();
- }
- }
- }
- super.tryToGetInitialLeasePeriod();
- }
-
- @Override
- public void tryToRenewLeasePeriod() throws Exception {
- while (pausedInTryToRenewLeasePeriod) {
- synchronized (this) {
- try {
- LOG.info("{} paused in tryToRenewLeasePeriod()", this);
- this.wait();
- } catch (InterruptedException e) {
- LOG.error("Interrupted whilst paused");
- Thread.currentThread().interrupt();
- }
- }
- }
- super.tryToRenewLeasePeriod();
- }
-
- @Override
- public boolean stillInLeasePeriod() {
- while (pausedInStillInLeasePeriod) {
- synchronized (this) {
- try {
- LOG.info("{} paused in stillInLeasePeriod()", this);
- this.wait();
- } catch (InterruptedException e) {
- LOG.error("Interrupted whilst paused");
- Thread.currentThread().interrupt();
- }
- }
- }
- return super.stillInLeasePeriod();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper Methods to pause functionality
- // ----------------------------------------------------------------------------------------------------------------
-
- public synchronized void pausedInTryToGetInitialLeasePeriod() {
- pausedInTryToGetInitialLeasePeriod = true;
- this.notifyAll();
- }
-
- public synchronized void pausedInTryToRenewLeasePeriod() {
- pausedInTryToRenewLeasePeriod = true;
- this.notifyAll();
- }
-
- public synchronized void pausedInStillInLeasePeriod() {
- pausedInStillInLeasePeriod = true;
- this.notifyAll();
- }
-
- public synchronized void resume() {
- pausedInTryToGetInitialLeasePeriod = false;
- pausedInTryToRenewLeasePeriod = false;
- pausedInStillInLeasePeriod = false;
- this.notifyAll();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java b/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
deleted file mode 100644
index d8d1d2c..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/ProgrammableTSOServer.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.tso.ProgrammableTSOServer.Response.ResponseType;
-import org.jboss.netty.bootstrap.ServerBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.Channels;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.group.ChannelGroup;
-import org.jboss.netty.channel.group.DefaultChannelGroup;
-import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.net.InetSocketAddress;
-import java.nio.channels.ClosedChannelException;
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.concurrent.Executors;
-
-/**
- * Used in tests. Allows to program the set of responses returned by a TSO
- */
-public class ProgrammableTSOServer extends SimpleChannelHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(ProgrammableTSOServer.class);
-
- private ChannelFactory factory;
- private ChannelGroup channelGroup;
-
- private Queue<Response> responseQueue = new LinkedList<>();
-
- @Inject
- public ProgrammableTSOServer(int port) {
- // Setup netty listener
- factory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(new ThreadFactoryBuilder()
- .setNameFormat("boss-%d").build()), Executors.newCachedThreadPool(new ThreadFactoryBuilder()
- .setNameFormat("worker-%d").build()), (Runtime.getRuntime().availableProcessors() * 2 + 1) * 2);
-
- // Create the global ChannelGroup
- channelGroup = new DefaultChannelGroup(ProgrammableTSOServer.class.getName());
-
- ServerBootstrap bootstrap = new ServerBootstrap(factory);
- bootstrap.setPipelineFactory(new TSOChannelHandler.TSOPipelineFactory(this));
-
- // Add the parent channel to the group
- Channel channel = bootstrap.bind(new InetSocketAddress(port));
- channelGroup.add(channel);
-
- LOG.info("********** Dumb TSO Server running on port {} **********", port);
- }
-
- // ************************* Main interface for tests *********************
-
- /**
- * Allows to add response to the queue of responses
- *
- * @param r
- * the response to add
- */
- public void queueResponse(Response r) {
- responseQueue.add(r);
- }
-
- /**
- * Removes all the current responses in the queue
- */
- public void cleanResponses() {
- responseQueue.clear();
- }
-
- // ******************** End of Main interface for tests *******************
-
- @Override
- public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- channelGroup.add(ctx.getChannel());
- }
-
- @Override
- public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- channelGroup.remove(ctx.getChannel());
- }
-
- /**
- * Handle received messages
- */
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
- Object msg = e.getMessage();
- if (msg instanceof TSOProto.Request) {
- TSOProto.Request request = (TSOProto.Request) msg;
- Channel channel = ctx.getChannel();
- if (request.hasHandshakeRequest()) {
- checkHandshake(ctx, request.getHandshakeRequest());
- return;
- }
- if (!handshakeCompleted(ctx)) {
- LOG.info("handshake not completed");
- channel.close();
- }
-
- Response resp = responseQueue.poll();
- if (request.hasTimestampRequest()) {
- if (resp == null || resp.type != ResponseType.TIMESTAMP) {
- throw new IllegalStateException("Expecting TS response to send but got " + resp);
- }
- TimestampResponse tsResp = (TimestampResponse) resp;
- sendTimestampResponse(tsResp.startTS, channel);
- } else if (request.hasCommitRequest()) {
- if (resp == null) {
- throw new IllegalStateException("Expecting COMMIT response to send but got null");
- }
- switch (resp.type) {
- case COMMIT:
- CommitResponse commitResp = (CommitResponse) resp;
- sendCommitResponse(commitResp.heuristicDecissionRequired,
- commitResp.startTS,
- commitResp.commitTS,
- channel);
- break;
- case ABORT:
- AbortResponse abortResp = (AbortResponse) resp;
- sendAbortResponse(abortResp.startTS, channel);
- break;
- default:
- throw new IllegalStateException("Expecting COMMIT response to send but got " + resp.type);
- }
- } else {
- LOG.error("Invalid request {}", request);
- ctx.getChannel().close();
- }
- } else {
- LOG.error("Unknown message type", msg);
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
- if (e.getCause() instanceof ClosedChannelException) {
- return;
- }
- LOG.warn("TSOHandler: Unexpected exception from downstream.", e.getCause());
- Channels.close(e.getChannel());
- }
-
- private void checkHandshake(final ChannelHandlerContext ctx, TSOProto.HandshakeRequest request) {
- TSOProto.HandshakeResponse.Builder response = TSOProto.HandshakeResponse.newBuilder();
- if (request.hasClientCapabilities()) {
-
- response.setClientCompatible(true).setServerCapabilities(TSOProto.Capabilities.newBuilder().build());
- TSOChannelContext tsoCtx = new TSOChannelContext();
- tsoCtx.setHandshakeComplete();
- ctx.setAttachment(tsoCtx);
- } else {
- response.setClientCompatible(false);
- }
- ctx.getChannel().write(TSOProto.Response.newBuilder().setHandshakeResponse(response.build()).build());
- }
-
- private boolean handshakeCompleted(ChannelHandlerContext ctx) {
- Object o = ctx.getAttachment();
- if (o instanceof TSOChannelContext) {
- TSOChannelContext tsoCtx = (TSOChannelContext) o;
- return tsoCtx.getHandshakeComplete();
- }
- return false;
- }
-
- private void sendTimestampResponse(long startTimestamp, Channel c) {
- TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
- TSOProto.TimestampResponse.Builder respBuilder = TSOProto.TimestampResponse.newBuilder();
- respBuilder.setStartTimestamp(startTimestamp);
- builder.setTimestampResponse(respBuilder.build());
- c.write(builder.build());
- }
-
- private void sendCommitResponse(boolean makeHeuristicDecission, long startTimestamp, long commitTimestamp, Channel c) {
- TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
- TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
- if (makeHeuristicDecission) { // If the commit is ambiguous is due to a new master TSO
- commitBuilder.setMakeHeuristicDecision(true);
- }
- commitBuilder.setAborted(false).setStartTimestamp(startTimestamp).setCommitTimestamp(commitTimestamp);
- builder.setCommitResponse(commitBuilder.build());
- c.write(builder.build());
- }
-
- private void sendAbortResponse(long startTimestamp, Channel c) {
- TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
- TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
- commitBuilder.setAborted(true).setStartTimestamp(startTimestamp);
- builder.setCommitResponse(commitBuilder.build());
- c.write(builder.build());
- }
-
- private static class TSOChannelContext {
- boolean handshakeComplete;
-
- TSOChannelContext() {
- handshakeComplete = false;
- }
-
- boolean getHandshakeComplete() {
- return handshakeComplete;
- }
-
- void setHandshakeComplete() {
- handshakeComplete = true;
- }
- }
-
- public static class TimestampResponse extends Response {
-
- final long startTS;
-
- public TimestampResponse(long startTS) {
- super(ResponseType.TIMESTAMP);
- this.startTS = startTS;
- }
-
- }
-
- public static class CommitResponse extends Response {
-
- final boolean heuristicDecissionRequired;
- final long startTS;
- final long commitTS;
-
- public CommitResponse(boolean heuristicDecissionRequired, long startTS, long commitTS) {
- super(ResponseType.COMMIT);
- this.heuristicDecissionRequired = heuristicDecissionRequired;
- this.startTS = startTS;
- this.commitTS = commitTS;
- }
-
- }
-
- public static class AbortResponse extends Response {
-
- final long startTS;
-
- public AbortResponse(long startTS) {
- super(ResponseType.ABORT);
- this.startTS = startTS;
- }
-
- }
-
- abstract static class Response {
-
- enum ResponseType {
- TIMESTAMP, COMMIT, ABORT;
- }
-
- final ResponseType type;
-
- public Response(ResponseType type) {
- this.type = type;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java b/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
deleted file mode 100644
index b882779..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TSOMockModule.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.InMemoryCommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import com.yahoo.omid.tso.TimestampOracleImpl.InMemoryTimestampStorage;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-public class TSOMockModule extends AbstractModule {
-
- private final TSOServerConfig config;
-
- public TSOMockModule(TSOServerConfig config) {
- this.config = config;
- }
-
- @Override
- protected void configure() {
-
- bind(TSOChannelHandler.class).in(Singleton.class);
- bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
- bind(CommitTable.class).to(InMemoryCommitTable.class).in(Singleton.class);
- bind(TimestampStorage.class).to(InMemoryTimestampStorage.class).in(Singleton.class);
- bind(TimestampOracle.class).to(PausableTimestampOracle.class).in(Singleton.class);
- bind(Panicker.class).to(MockPanicker.class).in(Singleton.class);
-
- install(config.getLeaseModule());
- install(new DisruptorModule());
-
- }
-
- @Provides
- TSOServerConfig provideTSOServerConfig() {
- return config;
- }
-
- @Provides
- @Singleton
- MetricsRegistry provideMetricsRegistry() {
- return new NullMetricsProvider();
- }
-
- @Provides
- @Named(TSO_HOST_AND_PORT_KEY)
- String provideTSOHostAndPort() throws SocketException, UnknownHostException {
- return NetworkInterfaceUtils.getTSOHostAndPort(config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java b/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
deleted file mode 100644
index 7872589..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TSOServerConfigTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.testng.annotations.Test;
-
-public class TSOServerConfigTest {
-
- @Test
- public void testParsesOK() throws Exception {
- new TSOServerConfig("test-omid.yml");
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
deleted file mode 100644
index 74879b0..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestBatch.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import org.jboss.netty.channel.Channel;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.AssertJUnit;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-
-public class TestBatch {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestBatch.class);
-
- private static final int BATCH_SIZE = 1000;
- private MetricsRegistry metrics = new NullMetricsProvider();
-
- @Mock
- private Channel channel;
- @Mock
- private RetryProcessor retryProcessor;
- @Mock
- private ReplyProcessor replyProcessor;
-
- // The batch element to test
- private PersistenceProcessorImpl.Batch batch;
-
- @BeforeMethod(alwaysRun = true, timeOut = 30_000)
- public void initMocksAndComponents() {
- MockitoAnnotations.initMocks(this);
- batch = new PersistenceProcessorImpl.Batch(BATCH_SIZE);
- }
-
- @Test
- public void testBatchFunctionality() {
-
- // Required mocks
- Channel channel = Mockito.mock(Channel.class);
- ReplyProcessor replyProcessor = Mockito.mock(ReplyProcessor.class);
- RetryProcessor retryProcessor = Mockito.mock(RetryProcessor.class);
-
- // The batch element to test
- PersistenceProcessorImpl.Batch batch = new PersistenceProcessorImpl.Batch(BATCH_SIZE);
-
- // Test initial state is OK
- AssertJUnit.assertFalse("Batch shouldn't be full", batch.isFull());
- AssertJUnit.assertEquals("Num events should be 0", 0, batch.getNumEvents());
-
- // Test adding a single commit event is OK
- MonitoringContext monCtx = new MonitoringContext(metrics);
- monCtx.timerStart("commitPersistProcessor");
- batch.addCommit(0, 1, channel, monCtx);
- AssertJUnit.assertFalse("Batch shouldn't be full", batch.isFull());
- AssertJUnit.assertEquals("Num events should be 1", 1, batch.getNumEvents());
-
- // Test when filling the batch with events, batch is full
- for (int i = 0; i < (BATCH_SIZE - 1); i++) {
- if (i % 2 == 0) {
- monCtx = new MonitoringContext(metrics);
- monCtx.timerStart("timestampPersistProcessor");
- batch.addTimestamp(i, channel, monCtx);
- } else {
- monCtx = new MonitoringContext(metrics);
- monCtx.timerStart("commitPersistProcessor");
- batch.addCommit(i, i + 1, channel, monCtx);
- }
- }
- AssertJUnit.assertTrue("Batch should be full", batch.isFull());
- AssertJUnit.assertEquals("Num events should be " + BATCH_SIZE, BATCH_SIZE, batch.getNumEvents());
-
- // Test an exception is thrown when batch is full and a new element is going to be added
- try {
- monCtx = new MonitoringContext(metrics);
- monCtx.timerStart("commitPersistProcessor");
- batch.addCommit(0, 1, channel, new MonitoringContext(metrics));
- Assert.fail("Should throw an IllegalStateException");
- } catch (IllegalStateException e) {
- AssertJUnit.assertEquals("message returned doesn't match", "batch full", e.getMessage());
- LOG.debug("IllegalStateException catched properly");
- }
-
- // Test that sending replies empties the batch
- final boolean MASTER_INSTANCE = true;
- final boolean SHOULD_MAKE_HEURISTIC_DECISSION = true;
- batch.sendRepliesAndReset(replyProcessor, retryProcessor, MASTER_INSTANCE);
- verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
- .timestampResponse(anyLong(), any(Channel.class), any(MonitoringContext.class));
- verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
- .commitResponse(eq(!SHOULD_MAKE_HEURISTIC_DECISSION), anyLong(), anyLong(),
- any(Channel.class), any(MonitoringContext.class));
- AssertJUnit.assertFalse("Batch shouldn't be full", batch.isFull());
- AssertJUnit.assertEquals("Num events should be 0", 0, batch.getNumEvents());
-
- }
-
- @Test
- public void testBatchFunctionalityWhenMastershipIsLost() {
- Channel channel = Mockito.mock(Channel.class);
-
- // Fill the batch with events till full
- for (int i = 0; i < BATCH_SIZE; i++) {
- if (i % 2 == 0) {
- MonitoringContext monCtx = new MonitoringContext(metrics);
- monCtx.timerStart("timestampPersistProcessor");
- batch.addTimestamp(i, channel, monCtx);
- } else {
- MonitoringContext monCtx = new MonitoringContext(metrics);
- monCtx.timerStart("commitPersistProcessor");
- batch.addCommit(i, i + 1, channel, monCtx);
- }
- }
-
- // Test that sending replies empties the batch also when the replica
- // is NOT master and calls the ambiguousCommitResponse() method on the
- // reply processor
- final boolean MASTER_INSTANCE = true;
- final boolean SHOULD_MAKE_HEURISTIC_DECISSION = true;
- batch.sendRepliesAndReset(replyProcessor, retryProcessor, !MASTER_INSTANCE);
- verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
- .timestampResponse(anyLong(), any(Channel.class), any(MonitoringContext.class));
- verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
- .commitResponse(eq(SHOULD_MAKE_HEURISTIC_DECISSION), anyLong(), anyLong(), any(Channel.class), any(
- MonitoringContext.class));
- assertFalse(batch.isFull(), "Batch shouldn't be full");
- assertEquals(batch.getNumEvents(), 0, "Num events should be 0");
-
- }
-}
[20/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/LongCache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/LongCache.java b/tso-server/src/main/java/org/apache/omid/tso/LongCache.java
new file mode 100644
index 0000000..f1936f5
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/LongCache.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import java.util.Arrays;
+
+public class LongCache implements Cache {
+
+ public static final long RESET_VALUE = 0L;
+
+ private final long[] cache;
+ private final int size;
+ private final int associativity;
+
+ public LongCache(int size, int associativity) {
+ this.size = size;
+ this.cache = new long[2 * (size + associativity)];
+ this.associativity = associativity;
+ }
+
+ @Override
+ public void reset() {
+ Arrays.fill(cache, RESET_VALUE);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.omid.tso.Cache#set(long, long)
+ */
+ @Override
+ public long set(long key, long value) {
+ final int index = index(key);
+ int oldestIndex = 0;
+ long oldestValue = Long.MAX_VALUE;
+ for (int i = 0; i < associativity; ++i) {
+ int currIndex = 2 * (index + i);
+ if (cache[currIndex] == key) {
+ oldestValue = 0;
+ oldestIndex = currIndex;
+ break;
+ }
+ if (cache[currIndex + 1] <= oldestValue) {
+ oldestValue = cache[currIndex + 1];
+ oldestIndex = currIndex;
+ }
+ }
+ cache[oldestIndex] = key;
+ cache[oldestIndex + 1] = value;
+ return oldestValue;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.omid.tso.Cache#get(long)
+ */
+ @Override
+ public long get(long key) {
+ final int index = index(key);
+ for (int i = 0; i < associativity; ++i) {
+ int currIndex = 2 * (index + i);
+ if (cache[currIndex] == key) {
+ return cache[currIndex + 1];
+ }
+ }
+ return 0;
+ }
+
+ private int index(long hash) {
+ return (int) (Math.abs(hash) % size);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/MockPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/MockPanicker.java b/tso-server/src/main/java/org/apache/omid/tso/MockPanicker.java
new file mode 100644
index 0000000..8a1bb10
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/MockPanicker.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MockPanicker implements Panicker {
+ private static final Logger LOG = LoggerFactory.getLogger(MockPanicker.class);
+
+ @Override
+ public void panic(String reason, Throwable cause) {
+ LOG.error("PANICKING: {}", reason, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/MonitoringContext.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/MonitoringContext.java b/tso-server/src/main/java/org/apache/omid/tso/MonitoringContext.java
new file mode 100644
index 0000000..e230c4d
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/MonitoringContext.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Throwables;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.concurrent.NotThreadSafe;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.omid.metrics.MetricsUtils.name;
+
+@NotThreadSafe
+public class MonitoringContext {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MonitoringContext.class);
+
+ private volatile boolean flag;
+ private Map<String, Long> elapsedTimeMsMap = new HashMap<>();
+ private Map<String, Stopwatch> timers = new ConcurrentHashMap<>();
+ private MetricsRegistry metrics;
+
+ public MonitoringContext(MetricsRegistry metrics) {
+ this.metrics = metrics;
+ }
+
+ public void timerStart(String name) {
+ Stopwatch stopwatch = new Stopwatch();
+ stopwatch.start();
+ timers.put(name, stopwatch);
+ }
+
+ public void timerStop(String name) {
+ if (flag) {
+ LOG.warn("timerStop({}) called after publish. Measurement was ignored. {}", name, Throwables.getStackTraceAsString(new Exception()));
+ return;
+ }
+ Stopwatch activeStopwatch = timers.get(name);
+ if (activeStopwatch == null) {
+ throw new IllegalStateException(
+ String.format("There is no %s timer in the %s monitoring context.", name, this));
+ }
+ activeStopwatch.stop();
+ elapsedTimeMsMap.put(name, activeStopwatch.elapsedTime(TimeUnit.NANOSECONDS));
+ timers.remove(name);
+ }
+
+ public void publish() {
+ flag = true;
+ for (String name : elapsedTimeMsMap.keySet()) {
+ Long durationInNs = elapsedTimeMsMap.get(name);
+ metrics.timer(name("tso", name)).update(durationInNs);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/NetworkInterfaceUtils.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/NetworkInterfaceUtils.java b/tso-server/src/main/java/org/apache/omid/tso/NetworkInterfaceUtils.java
new file mode 100644
index 0000000..2a2a6af
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/NetworkInterfaceUtils.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.net.HostAndPort;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+
+final public class NetworkInterfaceUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NetworkInterfaceUtils.class);
+
+ /**
+ * Returns an <code>InetAddress</code> object encapsulating what is most
+ * likely the machine's LAN IP address.
+ * <p/>
+ * This method is intended for use as a replacement of JDK method
+ * <code>InetAddress.getLocalHost</code>, because that method is ambiguous
+ * on Linux systems. Linux systems enumerate the loopback network
+ * interface the same way as regular LAN network interfaces, but the JDK
+ * <code>InetAddress.getLocalHost</code> method does not specify the
+ * algorithm used to select the address returned under such circumstances,
+ * and will often return the loopback address, which is not valid for
+ * network communication. Details
+ * <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4665037">here</a>.
+ * <p/>
+ * This method will scan all IP addresses on a particular network interface
+ * specified as parameter on the host machine to determine the IP address
+ * most likely to be the machine's LAN address. If the machine has multiple
+ * IP addresses, this method will prefer a site-local IP address (e.g.
+ * 192.168.x.x or 10.10.x.x, usually IPv4) if the machine has one (and will
+ * return the first site-local address if the machine has more than one),
+ * but if the machine does not hold a site-local address, this method will
+ * return simply the first non-loopback address found (IPv4 or IPv6).
+ * <p/>
+ * If this method cannot find a non-loopback address using this selection
+ * algorithm, it will fall back to calling and returning the result of JDK
+ * method <code>InetAddress.getLocalHost()</code>.
+ * <p/>
+ *
+ * @param ifaceName
+ * The name of the network interface to extract the IP address
+ * from
+ * @throws UnknownHostException
+ * If the LAN address of the machine cannot be found.
+ */
+ static InetAddress getIPAddressFromNetworkInterface(String ifaceName)
+ throws SocketException, UnknownHostException {
+
+ NetworkInterface iface = NetworkInterface.getByName(ifaceName);
+ if (iface == null) {
+ throw new IllegalArgumentException(
+ "Network interface " + ifaceName + " not found");
+ }
+
+ InetAddress candidateAddress = null;
+ Enumeration<InetAddress> inetAddrs = iface.getInetAddresses();
+ while (inetAddrs.hasMoreElements()) {
+ InetAddress inetAddr = inetAddrs.nextElement();
+ if (!inetAddr.isLoopbackAddress()) {
+ if (inetAddr.isSiteLocalAddress()) {
+ return inetAddr; // Return non-loopback site-local address
+ } else if (candidateAddress == null) {
+ // Found non-loopback address, but not necessarily site-local
+ candidateAddress = inetAddr;
+ }
+ }
+ }
+
+ if (candidateAddress != null) {
+ // Site-local address not found, but found other non-loopback addr
+ // Server might have a non-site-local address assigned to its NIC
+ // (or might be running IPv6 which deprecates "site-local" concept)
+ return candidateAddress;
+ }
+
+ // At this point, we did not find a non-loopback address.
+ // Fall back to returning whatever InetAddress.getLocalHost() returns
+ InetAddress jdkSuppliedAddress = InetAddress.getLocalHost();
+ if (jdkSuppliedAddress == null) {
+ throw new UnknownHostException(
+ "InetAddress.getLocalHost() unexpectedly returned null.");
+ }
+ return jdkSuppliedAddress;
+ }
+
+ public static String getTSOHostAndPort(TSOServerConfig config) throws SocketException, UnknownHostException {
+
+ // Build TSO host:port string and validate it
+ final String tsoNetIfaceName = config.getNetworkIfaceName();
+ InetAddress addr = getIPAddressFromNetworkInterface(tsoNetIfaceName);
+ final int tsoPort = config.getPort();
+
+ String tsoHostAndPortAsString = "N/A";
+ try {
+ tsoHostAndPortAsString = HostAndPort.fromParts(addr.getHostAddress(), tsoPort).toString();
+ } catch (IllegalArgumentException e) {
+ LOG.error("Cannot parse TSO host:port string {}", tsoHostAndPortAsString);
+ throw e;
+ }
+ return tsoHostAndPortAsString;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/Panicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/Panicker.java b/tso-server/src/main/java/org/apache/omid/tso/Panicker.java
new file mode 100644
index 0000000..70c4a8b
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/Panicker.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+public interface Panicker {
+ public void panic(String reason, Throwable cause);
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/PausableTimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/PausableTimestampOracle.java b/tso-server/src/main/java/org/apache/omid/tso/PausableTimestampOracle.java
new file mode 100644
index 0000000..6f54e26
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/PausableTimestampOracle.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+
+public class PausableTimestampOracle extends TimestampOracleImpl {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PausableTimestampOracle.class);
+
+ private volatile boolean tsoPaused = false;
+
+ @Inject
+ public PausableTimestampOracle(MetricsRegistry metrics,
+ TimestampStorage tsStorage,
+ Panicker panicker) throws IOException {
+ super(metrics, tsStorage, panicker);
+ }
+
+ @Override
+ public long next() throws IOException {
+ while (tsoPaused) {
+ synchronized (this) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted whilst paused");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ return super.next();
+ }
+
+ public synchronized void pause() {
+ tsoPaused = true;
+ this.notifyAll();
+ }
+
+ public synchronized void resume() {
+ tsoPaused = false;
+ this.notifyAll();
+ }
+
+ public boolean isTSOPaused() {
+ return tsoPaused;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
new file mode 100644
index 0000000..b52b75d
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessor.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.jboss.netty.channel.Channel;
+
+interface PersistenceProcessor {
+ void persistCommit(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx);
+
+ void persistAbort(long startTimestamp, boolean isRetry, Channel c, MonitoringContext monCtx);
+
+ void persistTimestamp(long startTimestamp, Channel c, MonitoringContext monCtx);
+
+ void persistLowWatermark(long lowWatermark);
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
new file mode 100644
index 0000000..26a104c
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/PersistenceProcessorImpl.java
@@ -0,0 +1,431 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.lmax.disruptor.BatchEventProcessor;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.SequenceBarrier;
+import com.lmax.disruptor.TimeoutBlockingWaitStrategy;
+import com.lmax.disruptor.TimeoutHandler;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.Histogram;
+import org.apache.omid.metrics.Meter;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.Timer;
+import org.jboss.netty.channel.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.omid.metrics.MetricsUtils.name;
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+class PersistenceProcessorImpl
+ implements EventHandler<PersistenceProcessorImpl.PersistEvent>, PersistenceProcessor, TimeoutHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PersistenceProcessor.class);
+
+ private final String tsoHostAndPort;
+ private final LeaseManagement leaseManager;
+ final ReplyProcessor reply;
+ final RetryProcessor retryProc;
+ final CommitTable.Client commitTableClient;
+ final CommitTable.Writer writer;
+ final Panicker panicker;
+ final RingBuffer<PersistEvent> persistRing;
+ final Batch batch;
+ final Timer flushTimer;
+ final Histogram batchSizeHistogram;
+ final Meter timeoutMeter;
+ final int batchPersistTimeoutInMs;
+
+ long lastFlush = System.nanoTime();
+
+ @Inject
+ PersistenceProcessorImpl(TSOServerConfig config,
+ MetricsRegistry metrics,
+ @Named(TSO_HOST_AND_PORT_KEY) String tsoHostAndPort,
+ LeaseManagement leaseManager,
+ CommitTable commitTable,
+ ReplyProcessor reply,
+ RetryProcessor retryProc,
+ Panicker panicker)
+ throws IOException {
+
+ this(config,
+ metrics,
+ tsoHostAndPort,
+ new Batch(config.getMaxBatchSize()),
+ leaseManager,
+ commitTable,
+ reply,
+ retryProc,
+ panicker);
+
+ }
+
+ @VisibleForTesting
+ PersistenceProcessorImpl(TSOServerConfig config,
+ MetricsRegistry metrics,
+ String tsoHostAndPort,
+ Batch batch,
+ LeaseManagement leaseManager,
+ CommitTable commitTable,
+ ReplyProcessor reply,
+ RetryProcessor retryProc,
+ Panicker panicker)
+ throws IOException {
+
+ this.tsoHostAndPort = tsoHostAndPort;
+ this.batch = batch;
+ this.batchPersistTimeoutInMs = config.getBatchPersistTimeoutInMs();
+ this.leaseManager = leaseManager;
+ this.commitTableClient = commitTable.getClient();
+ this.writer = commitTable.getWriter();
+ this.reply = reply;
+ this.retryProc = retryProc;
+ this.panicker = panicker;
+
+ LOG.info("Creating the persist processor with batch size {}, and timeout {}ms",
+ config.getMaxBatchSize(), batchPersistTimeoutInMs);
+
+ flushTimer = metrics.timer(name("tso", "persist", "flush"));
+ batchSizeHistogram = metrics.histogram(name("tso", "persist", "batchsize"));
+ timeoutMeter = metrics.meter(name("tso", "persist", "timeout"));
+
+ // FIXME consider putting something more like a phased strategy here to avoid
+ // all the syscalls
+ final TimeoutBlockingWaitStrategy timeoutStrategy
+ = new TimeoutBlockingWaitStrategy(config.getBatchPersistTimeoutInMs(), TimeUnit.MILLISECONDS);
+
+ persistRing = RingBuffer.createSingleProducer(
+ PersistEvent.EVENT_FACTORY, 1 << 20, timeoutStrategy); // 2^20 entries in ringbuffer
+ SequenceBarrier persistSequenceBarrier = persistRing.newBarrier();
+ BatchEventProcessor<PersistEvent> persistProcessor = new BatchEventProcessor<>(
+ persistRing,
+ persistSequenceBarrier,
+ this);
+ persistRing.addGatingSequences(persistProcessor.getSequence());
+ persistProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
+
+ ExecutorService persistExec = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("persist-%d").build());
+ persistExec.submit(persistProcessor);
+
+ }
+
+ @Override
+ public void onEvent(PersistEvent event, long sequence, boolean endOfBatch) throws Exception {
+
+ switch (event.getType()) {
+ case COMMIT:
+ event.getMonCtx().timerStart("commitPersistProcessor");
+ // TODO: What happens when the IOException is thrown?
+ writer.addCommittedTransaction(event.getStartTimestamp(), event.getCommitTimestamp());
+ batch.addCommit(event.getStartTimestamp(), event.getCommitTimestamp(), event.getChannel(),
+ event.getMonCtx());
+ break;
+ case ABORT:
+ sendAbortOrIdentifyFalsePositive(event.getStartTimestamp(), event.isRetry(), event.getChannel(),
+ event.getMonCtx());
+ break;
+ case TIMESTAMP:
+ event.getMonCtx().timerStart("timestampPersistProcessor");
+ batch.addTimestamp(event.getStartTimestamp(), event.getChannel(), event.getMonCtx());
+ break;
+ }
+ if (batch.isFull() || endOfBatch) {
+ maybeFlushBatch();
+ }
+
+ }
+
+ private void sendAbortOrIdentifyFalsePositive(long startTimestamp, boolean isRetry, Channel channel,
+ MonitoringContext monCtx) {
+
+ if (!isRetry) {
+ reply.abortResponse(startTimestamp, channel, monCtx);
+ return;
+ }
+
+ // If is a retry, we must check if it is a already committed request abort.
+ // This can happen because a client could have missed the reply, so it
+ // retried the request after a timeout. So we added to the batch and when
+ // it's flushed we'll add events to the retry processor in order to check
+ // for false positive aborts. It needs to be done after the flush in case
+ // the commit has occurred but it hasn't been persisted yet.
+ batch.addUndecidedRetriedRequest(startTimestamp, channel, monCtx);
+ }
+
+ // no event has been received in the timeout period
+ @Override
+ public void onTimeout(final long sequence) {
+ maybeFlushBatch();
+ }
+
+ /**
+ * Flush the current batch if it's full, or the timeout has been elapsed since the last flush.
+ */
+ private void maybeFlushBatch() {
+ if (batch.isFull()) {
+ flush();
+ } else if ((System.nanoTime() - lastFlush) > TimeUnit.MILLISECONDS.toNanos(batchPersistTimeoutInMs)) {
+ timeoutMeter.mark();
+ flush();
+ }
+ }
+
+ @VisibleForTesting
+ synchronized void flush() {
+ lastFlush = System.nanoTime();
+
+ boolean areWeStillMaster = true;
+ if (!leaseManager.stillInLeasePeriod()) {
+ // The master TSO replica has changed, so we must inform the
+ // clients about it when sending the replies and avoid flushing
+ // the current batch of TXs
+ areWeStillMaster = false;
+ // We need also to clear the data in the buffer
+ writer.clearWriteBuffer();
+ LOG.trace("Replica {} lost mastership before flushing data", tsoHostAndPort);
+ } else {
+ try {
+ writer.flush();
+ } catch (IOException e) {
+ panicker.panic("Error persisting commit batch", e.getCause());
+ }
+ batchSizeHistogram.update(batch.getNumEvents());
+ if (!leaseManager.stillInLeasePeriod()) {
+ // If after flushing this TSO server is not the master
+ // replica we need inform the client about it
+ areWeStillMaster = false;
+ LOG.warn("Replica {} lost mastership after flushing data", tsoHostAndPort);
+ }
+ }
+ flushTimer.update((System.nanoTime() - lastFlush));
+ batch.sendRepliesAndReset(reply, retryProc, areWeStillMaster);
+
+ }
+
+ @Override
+ public void persistCommit(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) {
+ long seq = persistRing.next();
+ PersistEvent e = persistRing.get(seq);
+ PersistEvent.makePersistCommit(e, startTimestamp, commitTimestamp, c, monCtx);
+ persistRing.publish(seq);
+ }
+
+ @Override
+ public void persistAbort(long startTimestamp, boolean isRetry, Channel c, MonitoringContext monCtx) {
+ long seq = persistRing.next();
+ PersistEvent e = persistRing.get(seq);
+ PersistEvent.makePersistAbort(e, startTimestamp, isRetry, c, monCtx);
+ persistRing.publish(seq);
+ }
+
+ @Override
+ public void persistTimestamp(long startTimestamp, Channel c, MonitoringContext monCtx) {
+ long seq = persistRing.next();
+ PersistEvent e = persistRing.get(seq);
+ PersistEvent.makePersistTimestamp(e, startTimestamp, c, monCtx);
+ persistRing.publish(seq);
+ }
+
+ @Override
+ public void persistLowWatermark(long lowWatermark) {
+ try {
+ writer.updateLowWatermark(lowWatermark);
+ } catch (IOException e) {
+ LOG.error("Should not be thrown");
+ }
+ }
+
+ public static class Batch {
+
+ final PersistEvent[] events;
+ final int maxBatchSize;
+ int numEvents;
+
+ Batch(int maxBatchSize) {
+ assert (maxBatchSize > 0);
+ this.maxBatchSize = maxBatchSize;
+ events = new PersistEvent[maxBatchSize];
+ numEvents = 0;
+ for (int i = 0; i < maxBatchSize; i++) {
+ events[i] = new PersistEvent();
+ }
+ }
+
+ boolean isFull() {
+ assert (numEvents <= maxBatchSize);
+ return numEvents == maxBatchSize;
+ }
+
+ int getNumEvents() {
+ return numEvents;
+ }
+
+ void addCommit(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) {
+ if (isFull()) {
+ throw new IllegalStateException("batch full");
+ }
+ int index = numEvents++;
+ PersistEvent e = events[index];
+ PersistEvent.makePersistCommit(e, startTimestamp, commitTimestamp, c, monCtx);
+ }
+
+ void addUndecidedRetriedRequest(long startTimestamp, Channel c, MonitoringContext monCtx) {
+ if (isFull()) {
+ throw new IllegalStateException("batch full");
+ }
+ int index = numEvents++;
+ PersistEvent e = events[index];
+ // We mark the event as an ABORT retry to identify the events to send
+ // to the retry processor
+ PersistEvent.makePersistAbort(e, startTimestamp, true, c, monCtx);
+ }
+
+ void addTimestamp(long startTimestamp, Channel c, MonitoringContext monCtx) {
+ if (isFull()) {
+ throw new IllegalStateException("batch full");
+ }
+ int index = numEvents++;
+ PersistEvent e = events[index];
+ PersistEvent.makePersistTimestamp(e, startTimestamp, c, monCtx);
+ }
+
+ void sendRepliesAndReset(ReplyProcessor reply, RetryProcessor retryProc, boolean isTSOInstanceMaster) {
+ for (int i = 0; i < numEvents; i++) {
+ PersistEvent e = events[i];
+ switch (e.getType()) {
+ case TIMESTAMP:
+ e.getMonCtx().timerStop("timestampPersistProcessor");
+ reply.timestampResponse(e.getStartTimestamp(), e.getChannel(), e.getMonCtx());
+ break;
+ case COMMIT:
+ e.getMonCtx().timerStop("commitPersistProcessor");
+ if (isTSOInstanceMaster) {
+ reply.commitResponse(false, e.getStartTimestamp(), e.getCommitTimestamp(), e.getChannel(),
+ e.getMonCtx());
+ } else {
+ // The client will need to perform heuristic actions to determine the output
+ reply.commitResponse(true, e.getStartTimestamp(), e.getCommitTimestamp(), e.getChannel(),
+ e.getMonCtx());
+ }
+ break;
+ case ABORT:
+ if (e.isRetry()) {
+ retryProc.disambiguateRetryRequestHeuristically(e.getStartTimestamp(), e.getChannel(),
+ e.getMonCtx());
+ } else {
+ LOG.error("We should not be receiving non-retried aborted requests in here");
+ }
+ break;
+ default:
+ LOG.error("We should receive only COMMIT or ABORT event types. Received {}", e.getType());
+ break;
+ }
+ }
+ numEvents = 0;
+ }
+
+ }
+
+ public final static class PersistEvent {
+
+ private MonitoringContext monCtx;
+
+ enum Type {
+ TIMESTAMP, COMMIT, ABORT
+ }
+
+ private Type type = null;
+ private Channel channel = null;
+
+ private boolean isRetry = false;
+ private long startTimestamp = 0;
+ private long commitTimestamp = 0;
+
+ static void makePersistCommit(PersistEvent e, long startTimestamp, long commitTimestamp, Channel c,
+ MonitoringContext monCtx) {
+ e.type = Type.COMMIT;
+ e.startTimestamp = startTimestamp;
+ e.commitTimestamp = commitTimestamp;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ static void makePersistAbort(PersistEvent e, long startTimestamp, boolean isRetry, Channel c,
+ MonitoringContext monCtx) {
+ e.type = Type.ABORT;
+ e.startTimestamp = startTimestamp;
+ e.isRetry = isRetry;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ static void makePersistTimestamp(PersistEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
+ e.type = Type.TIMESTAMP;
+ e.startTimestamp = startTimestamp;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ MonitoringContext getMonCtx() {
+ return monCtx;
+ }
+
+ Type getType() {
+ return type;
+ }
+
+ Channel getChannel() {
+ return channel;
+ }
+
+ boolean isRetry() {
+ return isRetry;
+ }
+
+ long getStartTimestamp() {
+ return startTimestamp;
+ }
+
+ long getCommitTimestamp() {
+ return commitTimestamp;
+ }
+
+ public final static EventFactory<PersistEvent> EVENT_FACTORY = new EventFactory<PersistEvent>() {
+ @Override
+ public PersistEvent newInstance() {
+ return new PersistEvent();
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
new file mode 100644
index 0000000..78a8bc4
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessor.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.jboss.netty.channel.Channel;
+
+interface ReplyProcessor {
+ /**
+ * Informs the client about the outcome of the Tx it was trying to
+ * commit. If the heuristic decision flat is enabled, the client
+ * will need to do additional actions for learning the final outcome.
+ *
+ * @param makeHeuristicDecision
+ * informs about whether heuristic actions are needed or not
+ * @param startTimestamp
+ * the start timestamp of the transaction (a.k.a. tx id)
+ * @param commitTimestamp
+ * the commit timestamp of the transaction
+ * @param channel
+ * the communication channed with the client
+ */
+ void commitResponse(boolean makeHeuristicDecision, long startTimestamp, long commitTimestamp, Channel channel, MonitoringContext monCtx);
+
+ void abortResponse(long startTimestamp, Channel c, MonitoringContext monCtx);
+
+ void timestampResponse(long startTimestamp, Channel c, MonitoringContext monCtx);
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
new file mode 100644
index 0000000..4e2bab4
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/ReplyProcessorImpl.java
@@ -0,0 +1,236 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.lmax.disruptor.BatchEventProcessor;
+import com.lmax.disruptor.BusySpinWaitStrategy;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.SequenceBarrier;
+import org.apache.omid.metrics.Meter;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.proto.TSOProto;
+import org.jboss.netty.channel.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.omid.metrics.MetricsUtils.name;
+
+class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyEvent>, ReplyProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ReplyProcessorImpl.class);
+
+ final RingBuffer<ReplyEvent> replyRing;
+ final Meter abortMeter;
+ final Meter commitMeter;
+ final Meter timestampMeter;
+
+ @Inject
+ ReplyProcessorImpl(MetricsRegistry metrics, Panicker panicker) {
+ replyRing = RingBuffer.<ReplyEvent>createMultiProducer(ReplyEvent.EVENT_FACTORY, 1 << 12,
+ new BusySpinWaitStrategy());
+ SequenceBarrier replySequenceBarrier = replyRing.newBarrier();
+ BatchEventProcessor<ReplyEvent> replyProcessor = new BatchEventProcessor<ReplyEvent>(
+ replyRing, replySequenceBarrier, this);
+ replyProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
+
+ replyRing.addGatingSequences(replyProcessor.getSequence());
+
+ ExecutorService replyExec = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("reply-%d").build());
+ replyExec.submit(replyProcessor);
+
+ abortMeter = metrics.meter(name("tso", "aborts"));
+ commitMeter = metrics.meter(name("tso", "commits"));
+ timestampMeter = metrics.meter(name("tso", "timestampAllocation"));
+ }
+
+ public void onEvent(ReplyEvent event, long sequence, boolean endOfBatch) throws Exception {
+ String name = null;
+ try {
+ switch (event.getType()) {
+ case COMMIT:
+ name = "commitReplyProcessor";
+ event.getMonCtx().timerStart(name);
+ handleCommitResponse(false, event.getStartTimestamp(), event.getCommitTimestamp(), event.getChannel());
+ break;
+ case HEURISTIC_COMMIT:
+ name = "commitReplyProcessor";
+ event.getMonCtx().timerStart(name);
+ handleCommitResponse(true, event.getStartTimestamp(), event.getCommitTimestamp(), event.getChannel());
+ break;
+ case ABORT:
+ name = "abortReplyProcessor";
+ event.getMonCtx().timerStart(name);
+ handleAbortResponse(event.getStartTimestamp(), event.getChannel());
+ break;
+ case TIMESTAMP:
+ name = "timestampReplyProcessor";
+ event.getMonCtx().timerStart(name);
+ handleTimestampResponse(event.getStartTimestamp(), event.getChannel());
+ break;
+ default:
+ LOG.error("Unknown event {}", event.getType());
+ break;
+ }
+ } finally {
+ if (name != null) {
+ event.getMonCtx().timerStop(name);
+ }
+ }
+ event.getMonCtx().publish();
+ }
+
+ @Override
+ public void commitResponse(boolean makeHeuristicDecision, long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) {
+ long seq = replyRing.next();
+ ReplyEvent e = replyRing.get(seq);
+ ReplyEvent.makeCommitResponse(makeHeuristicDecision, e, startTimestamp, commitTimestamp, c, monCtx);
+ replyRing.publish(seq);
+ }
+
+ @Override
+ public void abortResponse(long startTimestamp, Channel c, MonitoringContext monCtx) {
+ long seq = replyRing.next();
+ ReplyEvent e = replyRing.get(seq);
+ ReplyEvent.makeAbortResponse(e, startTimestamp, c, monCtx);
+ replyRing.publish(seq);
+ }
+
+ @Override
+ public void timestampResponse(long startTimestamp, Channel c, MonitoringContext monCtx) {
+ long seq = replyRing.next();
+ ReplyEvent e = replyRing.get(seq);
+ ReplyEvent.makeTimestampReponse(e, startTimestamp, c, monCtx);
+ replyRing.publish(seq);
+ }
+
+ void handleCommitResponse(boolean makeHeuristicDecision, long startTimestamp, long commitTimestamp, Channel c) {
+ TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
+ TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
+ if (makeHeuristicDecision) { // If the commit is ambiguous is due to a new master TSO
+ commitBuilder.setMakeHeuristicDecision(true);
+ }
+ commitBuilder.setAborted(false)
+ .setStartTimestamp(startTimestamp)
+ .setCommitTimestamp(commitTimestamp);
+ builder.setCommitResponse(commitBuilder.build());
+ c.write(builder.build());
+
+ commitMeter.mark();
+ }
+
+ void handleAbortResponse(long startTimestamp, Channel c) {
+ TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
+ TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
+ commitBuilder.setAborted(true)
+ .setStartTimestamp(startTimestamp);
+ builder.setCommitResponse(commitBuilder.build());
+ c.write(builder.build());
+
+ abortMeter.mark();
+ }
+
+ void handleTimestampResponse(long startTimestamp, Channel c) {
+ TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
+ TSOProto.TimestampResponse.Builder respBuilder = TSOProto.TimestampResponse.newBuilder();
+ respBuilder.setStartTimestamp(startTimestamp);
+ builder.setTimestampResponse(respBuilder.build());
+ c.write(builder.build());
+
+ timestampMeter.mark();
+ }
+
+ public final static class ReplyEvent {
+
+ enum Type {
+ TIMESTAMP, COMMIT, HEURISTIC_COMMIT, ABORT
+ }
+
+ private Type type = null;
+ private Channel channel = null;
+
+ private long startTimestamp = 0;
+ private long commitTimestamp = 0;
+ private MonitoringContext monCtx;
+
+ Type getType() {
+ return type;
+ }
+
+ Channel getChannel() {
+ return channel;
+ }
+
+ long getStartTimestamp() {
+ return startTimestamp;
+ }
+
+ long getCommitTimestamp() {
+ return commitTimestamp;
+ }
+
+ MonitoringContext getMonCtx() {
+ return monCtx;
+ }
+
+ static void makeTimestampReponse(ReplyEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
+ e.type = Type.TIMESTAMP;
+ e.startTimestamp = startTimestamp;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ static void makeCommitResponse(boolean makeHeuristicDecision, ReplyEvent e, long startTimestamp,
+ long commitTimestamp, Channel c, MonitoringContext monCtx) {
+
+ if (makeHeuristicDecision) {
+ e.type = Type.HEURISTIC_COMMIT;
+ } else {
+ e.type = Type.COMMIT;
+ }
+ e.startTimestamp = startTimestamp;
+ e.commitTimestamp = commitTimestamp;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ static void makeAbortResponse(ReplyEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
+ e.type = Type.ABORT;
+ e.startTimestamp = startTimestamp;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ public final static EventFactory<ReplyEvent> EVENT_FACTORY = new EventFactory<ReplyEvent>() {
+ @Override
+ public ReplyEvent newInstance() {
+ return new ReplyEvent();
+ }
+ };
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/RequestProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/RequestProcessor.java b/tso-server/src/main/java/org/apache/omid/tso/RequestProcessor.java
new file mode 100644
index 0000000..d2583ba
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/RequestProcessor.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.jboss.netty.channel.Channel;
+
+import java.util.Collection;
+
+// NOTE: public is required explicitly in the interface definition for Guice injection
+public interface RequestProcessor extends TSOStateManager.StateObserver {
+
+ void timestampRequest(Channel c, MonitoringContext monCtx);
+
+ void commitRequest(long startTimestamp, Collection<Long> writeSet, boolean isRetry, Channel c, MonitoringContext monCtx);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
new file mode 100644
index 0000000..6e40d77
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.lmax.disruptor.BatchEventProcessor;
+import com.lmax.disruptor.BusySpinWaitStrategy;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.SequenceBarrier;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.tso.TSOStateManager.TSOState;
+import org.jboss.netty.channel.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class RequestProcessorImpl implements EventHandler<RequestProcessorImpl.RequestEvent>, RequestProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RequestProcessorImpl.class);
+
+ private final TimestampOracle timestampOracle;
+ public final CommitHashMap hashmap;
+ private final MetricsRegistry metrics;
+ private final PersistenceProcessor persistProc;
+ private final RingBuffer<RequestEvent> requestRing;
+ private long lowWatermark = -1L;
+ private long epoch = -1L;
+
+ @Inject
+ RequestProcessorImpl(TSOServerConfig config,
+ MetricsRegistry metrics,
+ TimestampOracle timestampOracle,
+ PersistenceProcessor persistProc,
+ Panicker panicker) throws IOException {
+
+ this.metrics = metrics;
+
+ this.persistProc = persistProc;
+ this.timestampOracle = timestampOracle;
+
+ this.hashmap = new CommitHashMap(config.getMaxItems());
+
+ // Set up the disruptor thread
+ requestRing = RingBuffer.<RequestEvent>createMultiProducer(RequestEvent.EVENT_FACTORY, 1 << 12,
+ new BusySpinWaitStrategy());
+ SequenceBarrier requestSequenceBarrier = requestRing.newBarrier();
+ BatchEventProcessor<RequestEvent> requestProcessor =
+ new BatchEventProcessor<RequestEvent>(requestRing,
+ requestSequenceBarrier,
+ this);
+ requestRing.addGatingSequences(requestProcessor.getSequence());
+ requestProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
+
+ ExecutorService requestExec = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("request-%d").build());
+ // Each processor runs on a separate thread
+ requestExec.submit(requestProcessor);
+
+ }
+
+ /**
+ * This should be called when the TSO gets initialized or gets leadership
+ */
+ @Override
+ public void update(TSOState state) throws IOException {
+ LOG.info("Reseting RequestProcessor...");
+ this.lowWatermark = state.getLowWatermark();
+ persistProc.persistLowWatermark(lowWatermark);
+ this.epoch = state.getEpoch();
+ hashmap.reset();
+ LOG.info("RequestProcessor initialized with LWMs {} and Epoch {}", lowWatermark, epoch);
+ }
+
+ @Override
+ public void onEvent(RequestEvent event, long sequence, boolean endOfBatch) throws Exception {
+ String name = null;
+ try {
+ if (event.getType() == RequestEvent.Type.TIMESTAMP) {
+ name = "timestampReqProcessor";
+ event.getMonCtx().timerStart(name);
+ handleTimestamp(event);
+ } else if (event.getType() == RequestEvent.Type.COMMIT) {
+ name = "commitReqProcessor";
+ event.getMonCtx().timerStart(name);
+ handleCommit(event);
+ }
+ } finally {
+ if (name != null) {
+ event.getMonCtx().timerStop(name);
+ }
+ }
+
+ }
+
+ @Override
+ public void timestampRequest(Channel c, MonitoringContext monCtx) {
+ long seq = requestRing.next();
+ RequestEvent e = requestRing.get(seq);
+ RequestEvent.makeTimestampRequest(e, c, monCtx);
+ requestRing.publish(seq);
+ }
+
+ @Override
+ public void commitRequest(long startTimestamp, Collection<Long> writeSet, boolean isRetry, Channel c, MonitoringContext monCtx) {
+ long seq = requestRing.next();
+ RequestEvent e = requestRing.get(seq);
+ RequestEvent.makeCommitRequest(e, startTimestamp, monCtx, writeSet, isRetry, c);
+ requestRing.publish(seq);
+ }
+
+ public void handleTimestamp(RequestEvent requestEvent) {
+ long timestamp;
+
+ try {
+ timestamp = timestampOracle.next();
+ } catch (IOException e) {
+ LOG.error("Error getting timestamp", e);
+ return;
+ }
+
+ persistProc.persistTimestamp(timestamp, requestEvent.getChannel(), requestEvent.getMonCtx());
+ }
+
+ public long handleCommit(RequestEvent event) {
+ long startTimestamp = event.getStartTimestamp();
+ Iterable<Long> writeSet = event.writeSet();
+ boolean isRetry = event.isRetry();
+ Channel c = event.getChannel();
+
+ boolean committed = false;
+ long commitTimestamp = 0L;
+
+ int numCellsInWriteset = 0;
+ // 0. check if it should abort
+ if (startTimestamp <= lowWatermark) {
+ committed = false;
+ } else {
+ // 1. check the write-write conflicts
+ committed = true;
+ for (long cellId : writeSet) {
+ long value = hashmap.getLatestWriteForCell(cellId);
+ if (value != 0 && value >= startTimestamp) {
+ committed = false;
+ break;
+ }
+ numCellsInWriteset++;
+ }
+ }
+
+ if (committed) {
+ // 2. commit
+ try {
+ commitTimestamp = timestampOracle.next();
+
+ if (numCellsInWriteset > 0) {
+ long newLowWatermark = lowWatermark;
+
+ for (long r : writeSet) {
+ long removed = hashmap.putLatestWriteForCell(r, commitTimestamp);
+ newLowWatermark = Math.max(removed, newLowWatermark);
+ }
+
+ if (newLowWatermark != lowWatermark) {
+ LOG.trace("Setting new low Watermark to {}", newLowWatermark);
+ lowWatermark = newLowWatermark;
+ persistProc.persistLowWatermark(newLowWatermark);
+ }
+ }
+ persistProc.persistCommit(startTimestamp, commitTimestamp, c, event.getMonCtx());
+ } catch (IOException e) {
+ LOG.error("Error committing", e);
+ }
+ } else { // add it to the aborted list
+ persistProc.persistAbort(startTimestamp, isRetry, c, event.getMonCtx());
+ }
+
+ return commitTimestamp;
+ }
+
+ final static class RequestEvent implements Iterable<Long> {
+
+ enum Type {
+ TIMESTAMP, COMMIT
+ }
+
+ ;
+
+ private Type type = null;
+ private Channel channel = null;
+
+ private boolean isRetry = false;
+ private long startTimestamp = 0;
+ private MonitoringContext monCtx;
+ private long numCells = 0;
+
+ private static final int MAX_INLINE = 40;
+ private Long writeSet[] = new Long[MAX_INLINE];
+ private Collection<Long> writeSetAsCollection = null; // for the case where there's more than MAX_INLINE
+
+ static void makeTimestampRequest(RequestEvent e, Channel c, MonitoringContext monCtx) {
+ e.type = Type.TIMESTAMP;
+ e.channel = c;
+ e.monCtx = monCtx;
+ }
+
+ static void makeCommitRequest(RequestEvent e,
+ long startTimestamp, MonitoringContext monCtx, Collection<Long> writeSet,
+ boolean isRetry, Channel c) {
+ e.monCtx = monCtx;
+ e.type = Type.COMMIT;
+ e.channel = c;
+ e.startTimestamp = startTimestamp;
+ e.isRetry = isRetry;
+ if (writeSet.size() > MAX_INLINE) {
+ e.numCells = writeSet.size();
+ e.writeSetAsCollection = writeSet;
+ } else {
+ e.writeSetAsCollection = null;
+ e.numCells = writeSet.size();
+ int i = 0;
+ for (Long cellId : writeSet) {
+ e.writeSet[i] = cellId;
+ i++;
+ }
+ }
+ }
+
+ MonitoringContext getMonCtx() {
+ return monCtx;
+ }
+
+ Type getType() {
+ return type;
+ }
+
+ long getStartTimestamp() {
+ return startTimestamp;
+ }
+
+ Channel getChannel() {
+ return channel;
+ }
+
+ @Override
+ public Iterator<Long> iterator() {
+ if (writeSetAsCollection != null) {
+ return writeSetAsCollection.iterator();
+ }
+ return new Iterator<Long>() {
+ int i = 0;
+
+ @Override
+ public boolean hasNext() {
+ return i < numCells;
+ }
+
+ @Override
+ public Long next() {
+ return writeSet[i++];
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ Iterable<Long> writeSet() {
+ return this;
+ }
+
+ boolean isRetry() {
+ return isRetry;
+ }
+
+ public final static EventFactory<RequestEvent> EVENT_FACTORY
+ = new EventFactory<RequestEvent>() {
+ @Override
+ public RequestEvent newInstance() {
+ return new RequestEvent();
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/RetryProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/RetryProcessor.java b/tso-server/src/main/java/org/apache/omid/tso/RetryProcessor.java
new file mode 100644
index 0000000..9cc6a4b
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/RetryProcessor.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.jboss.netty.channel.Channel;
+
+interface RetryProcessor {
+ void disambiguateRetryRequestHeuristically(long startTimestamp, Channel c, MonitoringContext monCtx);
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/RetryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/RetryProcessorImpl.java b/tso-server/src/main/java/org/apache/omid/tso/RetryProcessorImpl.java
new file mode 100644
index 0000000..d1b9f68
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/RetryProcessorImpl.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.lmax.disruptor.BatchEventProcessor;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.SequenceBarrier;
+import com.lmax.disruptor.WaitStrategy;
+import com.lmax.disruptor.YieldingWaitStrategy;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.metrics.Meter;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.jboss.netty.channel.Channel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.omid.metrics.MetricsUtils.name;
+
+/**
+ * Manages the retry requests that clients can send when they did not received the response in the specified timeout
+ */
+class RetryProcessorImpl implements EventHandler<RetryProcessorImpl.RetryEvent>, RetryProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RetryProcessor.class);
+
+ // Disruptor chain stuff
+ final ReplyProcessor replyProc;
+ final RingBuffer<RetryEvent> retryRing;
+
+ final CommitTable.Client commitTableClient;
+ final CommitTable.Writer writer;
+
+ // Metrics
+ final Meter retriesMeter;
+
+ @Inject
+ RetryProcessorImpl(MetricsRegistry metrics, CommitTable commitTable, ReplyProcessor replyProc, Panicker panicker)
+ throws IOException
+ {
+
+ this.commitTableClient = commitTable.getClient();
+ this.writer = commitTable.getWriter();
+ this.replyProc = replyProc;
+
+ WaitStrategy strategy = new YieldingWaitStrategy();
+
+ retryRing = RingBuffer.createSingleProducer(RetryEvent.EVENT_FACTORY, 1 << 12, strategy);
+ SequenceBarrier retrySeqBarrier = retryRing.newBarrier();
+ BatchEventProcessor<RetryEvent> retryProcessor = new BatchEventProcessor<>(retryRing, retrySeqBarrier, this);
+ retryProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
+
+ retryRing.addGatingSequences(retryProcessor.getSequence());
+
+ ExecutorService retryExec = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("retry-%d").build());
+ retryExec.submit(retryProcessor);
+
+ // Metrics
+ retriesMeter = metrics.meter(name("tso", "retries"));
+ }
+
+ @Override
+ public void onEvent(final RetryEvent event, final long sequence, final boolean endOfBatch)
+ throws Exception {
+
+ switch (event.getType()) {
+ case COMMIT:
+ // TODO: What happens when the IOException is thrown?
+ handleCommitRetry(event);
+ break;
+ default:
+ assert (false);
+ break;
+ }
+
+ }
+
+ private void handleCommitRetry(RetryEvent event) throws InterruptedException, ExecutionException {
+
+ long startTimestamp = event.getStartTimestamp();
+
+ try {
+ Optional<CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(startTimestamp).get();
+ if (commitTimestamp.isPresent()) {
+ if (commitTimestamp.get().isValid()) {
+ LOG.trace("Valid commit TS found in Commit Table");
+ replyProc.commitResponse(false, startTimestamp, commitTimestamp.get().getValue(),
+ event.getChannel(), event.getMonCtx());
+ } else {
+ LOG.trace("Invalid commit TS found in Commit Table");
+ replyProc.abortResponse(startTimestamp, event.getChannel(), event.getMonCtx());
+ }
+ } else {
+ LOG.trace("No commit TS found in Commit Table");
+ replyProc.abortResponse(startTimestamp, event.getChannel(), event.getMonCtx());
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted reading from commit table");
+ Thread.currentThread().interrupt();
+ } catch (ExecutionException e) {
+ LOG.error("Error reading from commit table", e);
+ }
+
+ retriesMeter.mark();
+ }
+
+ @Override
+ public void disambiguateRetryRequestHeuristically(long startTimestamp, Channel c, MonitoringContext monCtx) {
+ long seq = retryRing.next();
+ RetryEvent e = retryRing.get(seq);
+ RetryEvent.makeCommitRetry(e, startTimestamp, c, monCtx);
+ retryRing.publish(seq);
+ }
+
+ public final static class RetryEvent {
+
+ enum Type {
+ COMMIT
+ }
+
+ private Type type = null;
+
+ private long startTimestamp = 0;
+ private Channel channel = null;
+ private MonitoringContext monCtx;
+
+ static void makeCommitRetry(RetryEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
+ e.monCtx = monCtx;
+ e.type = Type.COMMIT;
+ e.startTimestamp = startTimestamp;
+ e.channel = c;
+ }
+
+ MonitoringContext getMonCtx() {
+ return monCtx;
+ }
+
+ Type getType() {
+ return type;
+ }
+
+ Channel getChannel() {
+ return channel;
+ }
+
+ long getStartTimestamp() {
+ return startTimestamp;
+ }
+
+ public final static EventFactory<RetryEvent> EVENT_FACTORY
+ = new EventFactory<RetryEvent>() {
+ @Override
+ public RetryEvent newInstance() {
+ return new RetryEvent();
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/SystemExitPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/SystemExitPanicker.java b/tso-server/src/main/java/org/apache/omid/tso/SystemExitPanicker.java
new file mode 100644
index 0000000..927d2f8
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/SystemExitPanicker.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SystemExitPanicker implements Panicker {
+ private static final Logger LOG = LoggerFactory.getLogger(SystemExitPanicker.class);
+ private static final int PANIC_EXIT_CODE = 123;
+
+ @Override
+ public void panic(String reason, Throwable cause) {
+ LOG.error("PANICKING: {}", reason, cause);
+ System.exit(PANIC_EXIT_CODE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TSOChannelHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TSOChannelHandler.java b/tso-server/src/main/java/org/apache/omid/tso/TSOChannelHandler.java
new file mode 100644
index 0000000..3c1d6fe
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TSOChannelHandler.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.proto.TSOProto;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelHandler;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.ClosedChannelException;
+import java.util.concurrent.Executors;
+
+/**
+ * ChannelHandler for the TSO Server.
+ *
+ * Incoming requests are processed in this class
+ */
+public class TSOChannelHandler extends SimpleChannelHandler implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOChannelHandler.class);
+
+ private final ChannelFactory factory;
+
+ private final ServerBootstrap bootstrap;
+
+ @VisibleForTesting
+ Channel listeningChannel;
+ @VisibleForTesting
+ ChannelGroup channelGroup;
+
+ private RequestProcessor requestProcessor;
+
+ private TSOServerConfig config;
+
+ private MetricsRegistry metrics;
+
+ @Inject
+ public TSOChannelHandler(TSOServerConfig config, RequestProcessor requestProcessor, MetricsRegistry metrics) {
+
+ this.config = config;
+ this.metrics = metrics;
+ this.requestProcessor = requestProcessor;
+ // Setup netty listener
+ this.factory = new NioServerSocketChannelFactory(
+ Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("boss-%d").build()),
+ Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("worker-%d").build()),
+ (Runtime.getRuntime().availableProcessors() * 2 + 1) * 2);
+
+ this.bootstrap = new ServerBootstrap(factory);
+ bootstrap.setPipelineFactory(new TSOPipelineFactory(this));
+
+ }
+
+ /**
+ * Allows to create and connect the communication channel closing the previous one if existed
+ */
+ void reconnect() {
+ if (listeningChannel == null && channelGroup == null) {
+ LOG.debug("Creating communication channel...");
+ } else {
+ LOG.debug("Reconnecting communication channel...");
+ closeConnection();
+ }
+ // Create the global ChannelGroup
+ channelGroup = new DefaultChannelGroup(TSOChannelHandler.class.getName());
+ LOG.debug("\tCreating channel to listening for incoming connections in port {}", config.getPort());
+ listeningChannel = bootstrap.bind(new InetSocketAddress(config.getPort()));
+ channelGroup.add(listeningChannel);
+ LOG.debug("\tListening channel created and connected: {}", listeningChannel);
+ }
+
+ /**
+ * Allows to close the communication channel
+ */
+ void closeConnection() {
+ LOG.debug("Closing communication channel...");
+ if (listeningChannel != null) {
+ LOG.debug("\tUnbinding listening channel {}", listeningChannel);
+ listeningChannel.unbind().awaitUninterruptibly();
+ LOG.debug("\tListening channel {} unbound", listeningChannel);
+ }
+ if (channelGroup != null) {
+ LOG.debug("\tClosing channel group {}", channelGroup);
+ channelGroup.close().awaitUninterruptibly();
+ LOG.debug("\tChannel group {} closed", channelGroup);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Netty SimpleChannelHandler implementation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ channelGroup.add(ctx.getChannel());
+ LOG.debug("TSO channel connected: {}", ctx.getChannel());
+ }
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ channelGroup.remove(ctx.getChannel());
+ LOG.debug("TSO channel disconnected: {}", ctx.getChannel());
+ }
+
+ @Override
+ public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ LOG.debug("TSO channel closed: {}", ctx.getChannel());
+ }
+
+ /**
+ * Handle received messages
+ */
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
+ Object msg = e.getMessage();
+ if (msg instanceof TSOProto.Request) {
+ TSOProto.Request request = (TSOProto.Request) msg;
+ if (request.hasHandshakeRequest()) {
+ checkHandshake(ctx, request.getHandshakeRequest());
+ return;
+ }
+ if (!handshakeCompleted(ctx)) {
+ LOG.error("Handshake not completed. Closing channel {}", ctx.getChannel());
+ ctx.getChannel().close();
+ }
+
+ if (request.hasTimestampRequest()) {
+ requestProcessor.timestampRequest(ctx.getChannel(), new MonitoringContext(metrics));
+ } else if (request.hasCommitRequest()) {
+ TSOProto.CommitRequest cr = request.getCommitRequest();
+ requestProcessor.commitRequest(cr.getStartTimestamp(),
+ cr.getCellIdList(),
+ cr.getIsRetry(),
+ ctx.getChannel(),
+ new MonitoringContext(metrics));
+ } else {
+ LOG.error("Invalid request {}. Closing channel {}", request, ctx.getChannel());
+ ctx.getChannel().close();
+ }
+ } else {
+ LOG.error("Unknown message type", msg);
+ }
+ }
+
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
+ if (e.getCause() instanceof ClosedChannelException) {
+ LOG.warn("ClosedChannelException caught. Cause: ", e.getCause());
+ return;
+ }
+ LOG.warn("Unexpected exception from downstream. Closing channel {}", ctx.getChannel(), e.getCause());
+ ctx.getChannel().close();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Closeable implementation
+ // ----------------------------------------------------------------------------------------------------------------
+ @Override
+ public void close() throws IOException {
+ closeConnection();
+ factory.releaseExternalResources();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods and classes
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Contains the required context for handshake
+ */
+ private static class TSOChannelContext {
+
+ boolean handshakeComplete;
+
+ TSOChannelContext() {
+ handshakeComplete = false;
+ }
+
+ boolean getHandshakeComplete() {
+ return handshakeComplete;
+ }
+
+ void setHandshakeComplete() {
+ handshakeComplete = true;
+ }
+
+ }
+
+ private void checkHandshake(final ChannelHandlerContext ctx, TSOProto.HandshakeRequest request) {
+
+ TSOProto.HandshakeResponse.Builder response = TSOProto.HandshakeResponse.newBuilder();
+ if (request.hasClientCapabilities()) {
+
+ response.setClientCompatible(true)
+ .setServerCapabilities(TSOProto.Capabilities.newBuilder().build());
+ TSOChannelContext tsoCtx = new TSOChannelContext();
+ tsoCtx.setHandshakeComplete();
+ ctx.setAttachment(tsoCtx);
+ } else {
+ response.setClientCompatible(false);
+ }
+ ctx.getChannel().write(TSOProto.Response.newBuilder().setHandshakeResponse(response.build()).build());
+
+ }
+
+ private boolean handshakeCompleted(ChannelHandlerContext ctx) {
+
+ Object o = ctx.getAttachment();
+ if (o instanceof TSOChannelContext) {
+ TSOChannelContext tsoCtx = (TSOChannelContext) o;
+ return tsoCtx.getHandshakeComplete();
+ }
+ return false;
+
+ }
+
+ /**
+ * Netty pipeline configuration
+ */
+ static class TSOPipelineFactory implements ChannelPipelineFactory {
+
+ private final ChannelHandler handler;
+
+ TSOPipelineFactory(ChannelHandler handler) {
+ this.handler = handler;
+ }
+
+ public ChannelPipeline getPipeline() throws Exception {
+
+ ChannelPipeline pipeline = Channels.pipeline();
+ // Max packet length is 10MB. Transactions with so many cells
+ // that the packet is rejected will receive a ServiceUnavailableException.
+ // 10MB is enough for 2 million cells in a transaction though.
+ pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(10 * 1024 * 1024, 0, 4, 0, 4));
+ pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+ pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Request.getDefaultInstance()));
+ pipeline.addLast("protobufencoder", new ProtobufEncoder());
+ pipeline.addLast("handler", handler);
+
+ return pipeline;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/TSOModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/TSOModule.java b/tso-server/src/main/java/org/apache/omid/tso/TSOModule.java
new file mode 100644
index 0000000..52f88c7
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/TSOModule.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.name.Names;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+class TSOModule extends AbstractModule {
+ private final TSOServerConfig config;
+
+ TSOModule(TSOServerConfig config) {
+ this.config = config;
+ }
+
+ @Override
+ protected void configure() {
+
+ bind(TSOChannelHandler.class).in(Singleton.class);
+ bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
+ bind(TimestampOracle.class).to(TimestampOracleImpl.class).in(Singleton.class);
+ bind(Panicker.class).to(SystemExitPanicker.class).in(Singleton.class);
+
+ // Disruptor setup
+ install(new DisruptorModule());
+
+ }
+
+ @Provides
+ TSOServerConfig provideTSOServerConfig() {
+ return config;
+ }
+
+ @Provides
+ @Named(TSO_HOST_AND_PORT_KEY)
+ String provideTSOHostAndPort() throws SocketException, UnknownHostException {
+ return NetworkInterfaceUtils.getTSOHostAndPort(config);
+
+ }
+
+}
[23/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/TSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/TSOClient.java b/transaction-client/src/main/java/org/apache/omid/tso/client/TSOClient.java
new file mode 100644
index 0000000..590ec80
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/TSOClient.java
@@ -0,0 +1,933 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.base.Charsets;
+import com.google.common.net.HostAndPort;
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.proto.TSOProto;
+import org.apache.omid.zk.ZKUtils;
+import org.apache.statemachine.StateMachine;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.NodeCache;
+import org.apache.curator.framework.recipes.cache.NodeCacheListener;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timeout;
+import org.jboss.netty.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayDeque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Describes the abstract methods to communicate to the TSO server
+ */
+public class TSOClient implements TSOProtocol, NodeCacheListener {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOClient.class);
+
+ // Basic configuration constants & defaults TODO: Move DEFAULT_ZK_CLUSTER to a conf class???
+ public static final String DEFAULT_ZK_CLUSTER = "localhost:2181";
+
+ private static final long DEFAULT_EPOCH = -1L;
+ private volatile long epoch = DEFAULT_EPOCH;
+
+ // Attributes
+ private CuratorFramework zkClient;
+ private NodeCache currentTSOZNode;
+
+ private ChannelFactory factory;
+ private ClientBootstrap bootstrap;
+ private Channel currentChannel;
+ private final ScheduledExecutorService fsmExecutor;
+ StateMachine.Fsm fsm;
+
+ private final int requestTimeoutInMs;
+ private final int requestMaxRetries;
+ private final int tsoReconnectionDelayInSecs;
+ private InetSocketAddress tsoAddr;
+ private String zkCurrentTsoPath;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Construction
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public static TSOClient newInstance(OmidClientConfiguration tsoClientConf)
+ throws IOException, InterruptedException {
+ return new TSOClient(tsoClientConf);
+ }
+
+ // Avoid instantiation
+ private TSOClient(OmidClientConfiguration omidConf) throws IOException, InterruptedException {
+
+ // Start client with Nb of active threads = 3 as maximum.
+ int tsoExecutorThreads = omidConf.getExecutorThreads();
+
+ factory = new NioClientSocketChannelFactory(
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), tsoExecutorThreads);
+ // Create the bootstrap
+ bootstrap = new ClientBootstrap(factory);
+
+ requestTimeoutInMs = omidConf.getRequestTimeoutInMs();
+ requestMaxRetries = omidConf.getRequestMaxRetries();
+ tsoReconnectionDelayInSecs = omidConf.getReconnectionDelayInSecs();
+
+ LOG.info("Connecting to TSO...");
+ HostAndPort hp;
+ switch (omidConf.getConnectionType()) {
+ case HA:
+ zkClient = ZKUtils.initZKClient(omidConf.getConnectionString(),
+ omidConf.getZkNamespace(),
+ omidConf.getZkConnectionTimeoutInSecs());
+ zkCurrentTsoPath = omidConf.getZkCurrentTsoPath();
+ configureCurrentTSOServerZNodeCache(zkCurrentTsoPath);
+ String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
+ // TSO info includes the new TSO host:port address and epoch
+ String[] currentTSOAndEpochArray = tsoInfo.split("#");
+ hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
+ setTSOAddress(hp.getHostText(), hp.getPort());
+ epoch = Long.parseLong(currentTSOAndEpochArray[1]);
+ LOG.info("\t* Current TSO host:port found in ZK: {} Epoch {}", hp, getEpoch());
+ break;
+ case DIRECT:
+ default:
+ hp = HostAndPort.fromString(omidConf.getConnectionString());
+ setTSOAddress(hp.getHostText(), hp.getPort());
+ LOG.info("\t* TSO host:port {} will be connected directly", hp);
+ break;
+ }
+
+ fsmExecutor = Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder().setNameFormat("tsofsm-%d").build());
+ fsm = new StateMachine.FsmImpl(fsmExecutor);
+ fsm.setInitState(new DisconnectedState(fsm));
+
+ ChannelPipeline pipeline = bootstrap.getPipeline();
+ pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
+ pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+ pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
+ pipeline.addLast("protobufencoder", new ProtobufEncoder());
+ pipeline.addLast("handler", new Handler(fsm));
+
+ bootstrap.setOption("tcpNoDelay", true);
+ bootstrap.setOption("keepAlive", true);
+ bootstrap.setOption("reuseAddress", true);
+ bootstrap.setOption("connectTimeoutMillis", 100);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // TSOProtocol interface
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * @see TSOProtocol#getNewStartTimestamp()
+ */
+ @Override
+ public TSOFuture<Long> getNewStartTimestamp() {
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.TimestampRequest.Builder tsreqBuilder = TSOProto.TimestampRequest.newBuilder();
+ builder.setTimestampRequest(tsreqBuilder.build());
+ RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
+ fsm.sendEvent(request);
+ return new ForwardingTSOFuture<>(request);
+ }
+
+ /**
+ * @see TSOProtocol#commit(long, Set)
+ */
+ @Override
+ public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitbuilder = TSOProto.CommitRequest.newBuilder();
+ commitbuilder.setStartTimestamp(transactionId);
+ for (CellId cell : cells) {
+ commitbuilder.addCellId(cell.getCellId());
+ }
+ builder.setCommitRequest(commitbuilder.build());
+ RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
+ fsm.sendEvent(request);
+ return new ForwardingTSOFuture<>(request);
+ }
+
+ /**
+ * @see TSOProtocol#close()
+ */
+ @Override
+ public TSOFuture<Void> close() {
+ final CloseEvent closeEvent = new CloseEvent();
+ fsm.sendEvent(closeEvent);
+ closeEvent.addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ closeEvent.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ e.printStackTrace();
+ } catch (ExecutionException e) {
+ e.printStackTrace();
+ } finally {
+ fsmExecutor.shutdown();
+ if (currentTSOZNode != null) {
+ try {
+ currentTSOZNode.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ }
+
+ }
+ }, fsmExecutor);
+ return new ForwardingTSOFuture<>(closeEvent);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // High availability related interface
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Returns the epoch of the TSO server that initialized this transaction.
+ * Used for high availability support.
+ */
+ public long getEpoch() {
+ return epoch;
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // NodeCacheListener interface
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public void nodeChanged() throws Exception {
+
+ String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
+ // TSO info includes the new TSO host:port address and epoch
+ String[] currentTSOAndEpochArray = tsoInfo.split("#");
+ HostAndPort hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
+ setTSOAddress(hp.getHostText(), hp.getPort());
+ epoch = Long.parseLong(currentTSOAndEpochArray[1]);
+ LOG.info("CurrentTSO ZNode changed. New TSO Host & Port {}/Epoch {}", hp, getEpoch());
+ if (currentChannel != null && currentChannel.isConnected()) {
+ LOG.info("\tClosing channel with previous TSO {}", currentChannel);
+ currentChannel.close();
+ }
+
+ }
+
+ // ****************************************** Finite State Machine ************************************************
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // FSM: Events
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private static class ParamEvent<T> implements StateMachine.Event {
+
+ final T param;
+
+ ParamEvent(T param) {
+ this.param = param;
+ }
+
+ T getParam() {
+ return param;
+ }
+ }
+
+ private static class ErrorEvent extends ParamEvent<Throwable> {
+
+ ErrorEvent(Throwable t) {
+ super(t);
+ }
+ }
+
+ private static class ConnectedEvent extends ParamEvent<Channel> {
+
+ ConnectedEvent(Channel c) {
+ super(c);
+ }
+ }
+
+ private static class UserEvent<T> extends AbstractFuture<T>
+ implements StateMachine.DeferrableEvent {
+
+ void success(T value) {
+ set(value);
+ }
+
+ @Override
+ public void error(Throwable t) {
+ setException(t);
+ }
+ }
+
+ private static class CloseEvent extends UserEvent<Void> {
+
+ }
+
+ private static class ChannelClosedEvent extends ParamEvent<Throwable> {
+
+ ChannelClosedEvent(Throwable t) {
+ super(t);
+ }
+ }
+
+ private static class ReconnectEvent implements StateMachine.Event {
+
+ }
+
+ private static class HandshakeTimeoutEvent implements StateMachine.Event {
+
+ }
+
+ private static class TimestampRequestTimeoutEvent implements StateMachine.Event {
+
+ }
+
+ private static class CommitRequestTimeoutEvent implements StateMachine.Event {
+
+ final long startTimestamp;
+
+ CommitRequestTimeoutEvent(long startTimestamp) {
+ this.startTimestamp = startTimestamp;
+ }
+
+ public long getStartTimestamp() {
+ return startTimestamp;
+ }
+ }
+
+ private static class RequestEvent extends UserEvent<Long> {
+
+ TSOProto.Request req;
+ int retriesLeft;
+
+ RequestEvent(TSOProto.Request req, int retriesLeft) {
+ this.req = req;
+ this.retriesLeft = retriesLeft;
+ }
+
+ TSOProto.Request getRequest() {
+ return req;
+ }
+
+ void setRequest(TSOProto.Request request) {
+ this.req = request;
+ }
+
+ int getRetriesLeft() {
+ return retriesLeft;
+ }
+
+ void decrementRetries() {
+ retriesLeft--;
+ }
+
+ }
+
+ private static class ResponseEvent extends ParamEvent<TSOProto.Response> {
+
+ ResponseEvent(TSOProto.Response r) {
+ super(r);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // FSM: States
+ // ----------------------------------------------------------------------------------------------------------------
+
+ class BaseState extends StateMachine.State {
+
+ BaseState(StateMachine.Fsm fsm) {
+ super(fsm);
+ }
+
+ public StateMachine.State handleEvent(StateMachine.Event e) {
+ LOG.error("Unhandled event {} while in state {}", e, this.getClass().getName());
+ return this;
+ }
+ }
+
+ class DisconnectedState extends BaseState {
+
+ DisconnectedState(StateMachine.Fsm fsm) {
+ super(fsm);
+ LOG.debug("NEW STATE: DISCONNECTED");
+ }
+
+ public StateMachine.State handleEvent(RequestEvent e) {
+ fsm.deferEvent(e);
+ return tryToConnectToTSOServer();
+ }
+
+ public StateMachine.State handleEvent(CloseEvent e) {
+ factory.releaseExternalResources();
+ e.success(null);
+ return this;
+ }
+
+ private StateMachine.State tryToConnectToTSOServer() {
+ final InetSocketAddress tsoAddress = getAddress();
+ LOG.info("Trying to connect to TSO [{}]", tsoAddress);
+ ChannelFuture channelFuture = bootstrap.connect(tsoAddress);
+ channelFuture.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture channelFuture) throws Exception {
+ if (channelFuture.isSuccess()) {
+ LOG.info("Connection to TSO [{}] established. Channel {}",
+ tsoAddress, channelFuture.getChannel());
+ } else {
+ LOG.error("Failed connection attempt to TSO [{}] failed. Channel {}",
+ tsoAddress, channelFuture.getChannel());
+ }
+ }
+ });
+ return new ConnectingState(fsm);
+ }
+ }
+
+ private class ConnectingState extends BaseState {
+
+ ConnectingState(StateMachine.Fsm fsm) {
+ super(fsm);
+ LOG.debug("NEW STATE: CONNECTING");
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ fsm.deferEvent(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ConnectedEvent e) {
+ return new HandshakingState(fsm, e.getParam());
+ }
+
+ public StateMachine.State handleEvent(ChannelClosedEvent e) {
+ return new ConnectionFailedState(fsm, e.getParam());
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ return new ConnectionFailedState(fsm, e.getParam());
+ }
+
+ }
+
+ private static class RequestAndTimeout {
+
+ final RequestEvent event;
+ final Timeout timeout;
+
+ RequestAndTimeout(RequestEvent event, Timeout timeout) {
+ this.event = event;
+ this.timeout = timeout;
+ }
+
+ RequestEvent getRequest() {
+ return event;
+ }
+
+ Timeout getTimeout() {
+ return timeout;
+ }
+
+ public String toString() {
+ String info = "Request type ";
+ if (event.getRequest().hasTimestampRequest()) {
+ info += "[Timestamp]";
+ } else if (event.getRequest().hasCommitRequest()) {
+ info += "[Commit] Start TS ->" + event.getRequest().getCommitRequest().getStartTimestamp();
+ } else {
+ info += "NONE";
+ }
+ return info;
+ }
+ }
+
+ private class HandshakingState extends BaseState {
+
+ final Channel channel;
+
+ final HashedWheelTimer timeoutExecutor = new HashedWheelTimer(
+ new ThreadFactoryBuilder().setNameFormat("tso-client-timeout").build());
+ final Timeout timeout;
+
+ HandshakingState(StateMachine.Fsm fsm, Channel channel) {
+ super(fsm);
+ LOG.debug("NEW STATE: HANDSHAKING");
+ this.channel = channel;
+ TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
+ // Add the required handshake capabilities when necessary
+ handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
+ channel.write(TSOProto.Request.newBuilder().setHandshakeRequest(handshake.build()).build());
+ timeout = newTimeout();
+ }
+
+ private Timeout newTimeout() {
+ if (requestTimeoutInMs > 0) {
+ return timeoutExecutor.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) {
+ fsm.sendEvent(new HandshakeTimeoutEvent());
+ }
+ }, 30, TimeUnit.SECONDS);
+ } else {
+ return null;
+ }
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ fsm.deferEvent(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ResponseEvent e) {
+ if (e.getParam().hasHandshakeResponse() && e.getParam().getHandshakeResponse().getClientCompatible()) {
+ if (timeout != null) {
+ timeout.cancel();
+ }
+ return new ConnectedState(fsm, channel, timeoutExecutor);
+ } else {
+ cleanupState();
+ LOG.error("Client incompatible with server");
+ return new HandshakeFailedState(fsm, new HandshakeFailedException());
+ }
+ }
+
+ public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
+ cleanupState();
+ return new ClosingState(fsm);
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ cleanupState();
+ Throwable exception = e.getParam();
+ LOG.error("Error during handshake", exception);
+ return new HandshakeFailedState(fsm, exception);
+ }
+
+ private void cleanupState() {
+ timeoutExecutor.stop();
+ channel.close();
+ if (timeout != null) {
+ timeout.cancel();
+ }
+ }
+
+ }
+
+ class ConnectionFailedState extends BaseState {
+
+ final HashedWheelTimer reconnectionTimeoutExecutor = new HashedWheelTimer(
+ new ThreadFactoryBuilder().setNameFormat("tso-client-backoff-timeout").build());
+
+ Throwable exception;
+
+ ConnectionFailedState(final StateMachine.Fsm fsm, final Throwable exception) {
+ super(fsm);
+ LOG.debug("NEW STATE: CONNECTION FAILED [RE-CONNECTION BACKOFF]");
+ this.exception = exception;
+ reconnectionTimeoutExecutor.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) {
+ fsm.sendEvent(new ReconnectEvent());
+ }
+ }, tsoReconnectionDelayInSecs, TimeUnit.SECONDS);
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ e.error(exception);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ChannelClosedEvent e) {
+ return new DisconnectedState(fsm);
+ }
+
+ public StateMachine.State handleEvent(ReconnectEvent e) {
+ return new DisconnectedState(fsm);
+ }
+
+ }
+
+ private class HandshakeFailedState extends ConnectionFailedState {
+
+ HandshakeFailedState(StateMachine.Fsm fsm, Throwable exception) {
+ super(fsm, exception);
+ LOG.debug("STATE: HANDSHAKING FAILED");
+ }
+
+ }
+
+ class ConnectedState extends BaseState {
+
+ final Queue<RequestAndTimeout> timestampRequests;
+ final Map<Long, RequestAndTimeout> commitRequests;
+ final Channel channel;
+
+ final HashedWheelTimer timeoutExecutor;
+
+ ConnectedState(StateMachine.Fsm fsm, Channel channel, HashedWheelTimer timeoutExecutor) {
+ super(fsm);
+ LOG.debug("NEW STATE: CONNECTED");
+ this.channel = channel;
+ this.timeoutExecutor = timeoutExecutor;
+ timestampRequests = new ArrayDeque<>();
+ commitRequests = new HashMap<>();
+ }
+
+ private Timeout newTimeout(final StateMachine.Event timeoutEvent) {
+ if (requestTimeoutInMs > 0) {
+ return timeoutExecutor.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) {
+ fsm.sendEvent(timeoutEvent);
+ }
+ }, requestTimeoutInMs, TimeUnit.MILLISECONDS);
+ } else {
+ return null;
+ }
+ }
+
+ private void sendRequest(final StateMachine.Fsm fsm, RequestEvent request) {
+ TSOProto.Request req = request.getRequest();
+
+ if (req.hasTimestampRequest()) {
+ timestampRequests.add(new RequestAndTimeout(request, newTimeout(new TimestampRequestTimeoutEvent())));
+ } else if (req.hasCommitRequest()) {
+ TSOProto.CommitRequest commitReq = req.getCommitRequest();
+ commitRequests.put(commitReq.getStartTimestamp(), new RequestAndTimeout(
+ request, newTimeout(new CommitRequestTimeoutEvent(commitReq.getStartTimestamp()))));
+ } else {
+ request.error(new IllegalArgumentException("Unknown request type"));
+ return;
+ }
+ ChannelFuture f = channel.write(req);
+
+ f.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if (!future.isSuccess()) {
+ fsm.sendEvent(new ErrorEvent(future.getCause()));
+ }
+ }
+ });
+ }
+
+ private void handleResponse(ResponseEvent response) {
+ TSOProto.Response resp = response.getParam();
+ if (resp.hasTimestampResponse()) {
+ if (timestampRequests.size() == 0) {
+ LOG.debug("Received timestamp response when no requests outstanding");
+ return;
+ }
+ RequestAndTimeout e = timestampRequests.remove();
+ e.getRequest().success(resp.getTimestampResponse().getStartTimestamp());
+ if (e.getTimeout() != null) {
+ e.getTimeout().cancel();
+ }
+ } else if (resp.hasCommitResponse()) {
+ long startTimestamp = resp.getCommitResponse().getStartTimestamp();
+ RequestAndTimeout e = commitRequests.remove(startTimestamp);
+ if (e == null) {
+ LOG.debug("Received commit response for request that doesn't exist. Start TS: {}", startTimestamp);
+ return;
+ }
+ if (e.getTimeout() != null) {
+ e.getTimeout().cancel();
+ }
+ if (resp.getCommitResponse().getAborted()) {
+ e.getRequest().error(new AbortException());
+ } else {
+ // Check if the commit response received implies heuristic
+ // actions during commit (because there's a new TSO master
+ // replica) and inform the caller (e.g. the TxMgr) about it
+ if (resp.getCommitResponse().getMakeHeuristicDecision()) {
+ e.getRequest().error(new NewTSOException());
+ } else {
+ e.getRequest().success(resp.getCommitResponse().getCommitTimestamp());
+ }
+ }
+ }
+ }
+
+ public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
+ if (!timestampRequests.isEmpty()) {
+ RequestAndTimeout r = timestampRequests.remove();
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ }
+ return this;
+ }
+
+ public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
+ long startTimestamp = e.getStartTimestamp();
+ if (commitRequests.containsKey(startTimestamp)) {
+ RequestAndTimeout r = commitRequests.remove(startTimestamp);
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ }
+ return this;
+ }
+
+ public StateMachine.State handleEvent(CloseEvent e) {
+ LOG.debug("CONNECTED STATE: CloseEvent");
+ timeoutExecutor.stop();
+ closeChannelAndErrorRequests();
+ fsm.deferEvent(e);
+ return new ClosingState(fsm);
+ }
+
+ public StateMachine.State handleEvent(RequestEvent e) {
+ sendRequest(fsm, e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ResponseEvent e) {
+ handleResponse(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ LOG.debug("CONNECTED STATE: ErrorEvent");
+ timeoutExecutor.stop();
+ handleError(fsm);
+ return new ClosingState(fsm);
+ }
+
+ private void handleError(StateMachine.Fsm fsm) {
+ LOG.debug("CONNECTED STATE: Cancelling Timeouts in handleError");
+ while (timestampRequests.size() > 0) {
+ RequestAndTimeout r = timestampRequests.remove();
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ }
+ Iterator<Map.Entry<Long, RequestAndTimeout>> iter = commitRequests.entrySet().iterator();
+ while (iter.hasNext()) {
+ RequestAndTimeout r = iter.next().getValue();
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ iter.remove();
+ }
+ channel.close();
+ }
+
+ private void queueRetryOrError(StateMachine.Fsm fsm, RequestEvent e) {
+ if (e.getRetriesLeft() > 0) {
+ e.decrementRetries();
+ if (e.getRequest().hasCommitRequest()) {
+ TSOProto.CommitRequest commitRequest = e.getRequest().getCommitRequest();
+ if (!commitRequest.getIsRetry()) { // Create a new retry for the commit request
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
+ commitBuilder.mergeFrom(commitRequest);
+ commitBuilder.setIsRetry(true);
+ builder.setCommitRequest(commitBuilder.build());
+ e.setRequest(builder.build());
+ }
+ }
+ fsm.sendEvent(e);
+ } else {
+ e.error(
+ new ServiceUnavailableException("Number of retries exceeded. This API request failed permanently"));
+ }
+ }
+
+ private void closeChannelAndErrorRequests() {
+ channel.close();
+ for (RequestAndTimeout r : timestampRequests) {
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ r.getRequest().error(new ClosingException());
+ }
+ for (RequestAndTimeout r : commitRequests.values()) {
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ r.getRequest().error(new ClosingException());
+ }
+ }
+ }
+
+ private class ClosingState extends BaseState {
+
+ ClosingState(StateMachine.Fsm fsm) {
+ super(fsm);
+ LOG.debug("NEW STATE: CLOSING");
+ }
+
+ public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ResponseEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ fsm.deferEvent(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ChannelClosedEvent e) {
+ return new DisconnectedState(fsm);
+ }
+
+ public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
+ return this;
+ }
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper classes & methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private class Handler extends SimpleChannelHandler {
+
+ private StateMachine.Fsm fsm;
+
+ Handler(StateMachine.Fsm fsm) {
+ this.fsm = fsm;
+ }
+
+ @Override
+ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
+ currentChannel = e.getChannel();
+ LOG.debug("HANDLER (CHANNEL CONNECTED): Connection {}. Sending connected event to FSM", e);
+ fsm.sendEvent(new ConnectedEvent(e.getChannel()));
+ }
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ LOG.debug("HANDLER (CHANNEL DISCONNECTED): Connection {}. Sending error event to FSM", e);
+ fsm.sendEvent(new ErrorEvent(new ConnectionException()));
+ }
+
+ @Override
+ public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ LOG.debug("HANDLER (CHANNEL CLOSED): Connection {}. Sending channel closed event to FSM", e);
+ fsm.sendEvent(new ChannelClosedEvent(new ConnectionException()));
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
+ if (e.getMessage() instanceof TSOProto.Response) {
+ fsm.sendEvent(new ResponseEvent((TSOProto.Response) e.getMessage()));
+ } else {
+ LOG.warn("Received unknown message", e.getMessage());
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+ LOG.error("Error on channel {}", ctx.getChannel(), e.getCause());
+ fsm.sendEvent(new ErrorEvent(e.getCause()));
+ }
+ }
+
+ private synchronized void setTSOAddress(String host, int port) {
+ tsoAddr = new InetSocketAddress(host, port);
+ }
+
+ private synchronized InetSocketAddress getAddress() {
+ return tsoAddr;
+ }
+
+ private void configureCurrentTSOServerZNodeCache(String currentTsoPath) {
+ try {
+ currentTSOZNode = new NodeCache(zkClient, currentTsoPath);
+ currentTSOZNode.getListenable().addListener(this);
+ currentTSOZNode.start(true);
+ } catch (Exception e) {
+ throw new IllegalStateException("Cannot start watcher on current TSO Server ZNode: " + e.getMessage());
+ }
+ }
+
+ private String getCurrentTSOInfoFoundInZK(String currentTsoPath) {
+ ChildData currentTSOData = currentTSOZNode.getCurrentData();
+ if (currentTSOData == null) {
+ throw new IllegalStateException("No data found in ZKNode " + currentTsoPath);
+ }
+ byte[] currentTSOAndEpochAsBytes = currentTSOData.getData();
+ if (currentTSOAndEpochAsBytes == null) {
+ throw new IllegalStateException("No data found for current TSO in ZKNode " + currentTsoPath);
+ }
+ return new String(currentTSOAndEpochAsBytes, Charsets.UTF_8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/TSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/TSOFuture.java b/transaction-client/src/main/java/org/apache/omid/tso/client/TSOFuture.java
new file mode 100644
index 0000000..878bf30
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/TSOFuture.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+
+public interface TSOFuture<T> extends Future<T> {
+ public void addListener(Runnable listener, Executor executor);
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/TSOProtocol.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/TSOProtocol.java b/transaction-client/src/main/java/org/apache/omid/tso/client/TSOProtocol.java
new file mode 100644
index 0000000..548cea1
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/TSOProtocol.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import java.util.Set;
+
+/**
+ * Defines the protocol used on the client side to abstract communication to the TSO server
+ */
+public interface TSOProtocol {
+
+ /**
+ * Returns a new timestamp assigned by on the server-side
+ * @return the newly assigned timestamp as a future. If an error was detected, the future will contain a
+ * corresponding protocol exception
+ * @see TimestampOracle
+ * @see TSOServer
+ */
+ TSOFuture<Long> getNewStartTimestamp();
+
+ /**
+ * Returns the result of the conflict detection made on the server-side for the specified transaction
+ * @param transactionId
+ * the transaction to check for conflicts
+ * @param writeSet
+ * the writeSet of the transaction, which includes all the modified cells
+ * @return the commit timestamp as a future if the transaction was committed. If the transaction was aborted due
+ * to conflicts with a concurrent transaction, the future will include an AbortException. If an error was detected,
+ * the future will contain a corresponding protocol exception
+ * @see TimestampOracle
+ * @see TSOServer
+ */
+ TSOFuture<Long> commit(long transactionId, Set<? extends CellId> writeSet);
+
+ /**
+ * Closes the communication with the TSO server
+ * @return nothing. If an error was detected, the future will contain a corresponding protocol exception
+ */
+ TSOFuture<Void> close();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/util/DummyCellIdImpl.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/util/DummyCellIdImpl.java b/transaction-client/src/main/java/org/apache/omid/tso/util/DummyCellIdImpl.java
new file mode 100644
index 0000000..f28c241
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/util/DummyCellIdImpl.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.util;
+
+import org.apache.omid.tso.client.CellId;
+
+public class DummyCellIdImpl implements CellId {
+
+ private final long cellId;
+
+ public DummyCellIdImpl(long cellId) {
+ this.cellId = cellId;
+ }
+
+ @Override
+ public long getCellId() {
+ return cellId;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/resources/omid-client-config.yml
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/resources/omid-client-config.yml b/transaction-client/src/main/resources/omid-client-config.yml
index daed7d3..4263c35 100644
--- a/transaction-client/src/main/resources/omid-client-config.yml
+++ b/transaction-client/src/main/resources/omid-client-config.yml
@@ -7,13 +7,13 @@
# ---------------------------------------------------------------------------------------------------------------------
# Direct connection to host:port
-connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
+connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
connectionString: "localhost:54758"
# When Omid is working in High Availability mode, two or more replicas of the TSO server are running in primary/backup
# mode. When a TSO server replica is elected as master, it publishes its address through ZK. In order to configure
# the Omid client to access the TSO server in HA mode:
-# 1) set 'connectionType' to !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType HA
+# 1) set 'connectionType' to !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType HA
# 2) set 'connectionString' to the ZK cluster connection string where the server is publishing its address
zkConnectionTimeoutInSecs: 10
# In HA mode, make sure that the next settings match same settings on the TSO server side
@@ -36,4 +36,4 @@ executorThreads: 3
# Configure whether the TM performs the post-commit actions for a tx (update shadow cells and clean commit table entry)
# before returning to the control to the client (SYNC) or in parallel (ASYNC)
-postCommitMode: !!com.yahoo.omid.tso.client.OmidClientConfiguration$PostCommitMode SYNC
\ No newline at end of file
+postCommitMode: !!org.apache.omid.tso.client.OmidClientConfiguration$PostCommitMode SYNC
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java b/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java
deleted file mode 100644
index deca92f..0000000
--- a/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.collect.Sets;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.InMemoryCommitTable;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import org.testng.Assert;
-import org.testng.AssertJUnit;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.AssertJUnit.assertEquals;
-
-public class TestMockTSOClient {
-
- final static public CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
- final static public CellId c2 = new DummyCellIdImpl(-0xfeedcafeL);
-
- @Test(timeOut = 10000)
- public void testConflicts() throws Exception {
- CommitTable commitTable = new InMemoryCommitTable();
- TSOProtocol client = new MockTSOClient(commitTable.getWriter());
-
- long tr1 = client.getNewStartTimestamp().get();
- long tr2 = client.getNewStartTimestamp().get();
-
- client.commit(tr1, Sets.newHashSet(c1)).get();
-
- try {
- client.commit(tr2, Sets.newHashSet(c1, c2)).get();
- Assert.fail("Shouldn't have committed");
- } catch (ExecutionException ee) {
- assertEquals("Should have aborted", ee.getCause().getClass(), AbortException.class);
- }
- }
-
- @Test(timeOut = 10000)
- public void testWatermarkUpdate() throws Exception {
- CommitTable commitTable = new InMemoryCommitTable();
- TSOProtocol client = new MockTSOClient(commitTable.getWriter());
- CommitTable.Client commitTableClient = commitTable.getClient();
-
- long tr1 = client.getNewStartTimestamp().get();
- client.commit(tr1, Sets.newHashSet(c1)).get();
-
- long initWatermark = commitTableClient.readLowWatermark().get();
-
- long tr2 = client.getNewStartTimestamp().get();
- client.commit(tr2, Sets.newHashSet(c1)).get();
-
- long newWatermark = commitTableClient.readLowWatermark().get();
- AssertJUnit.assertTrue("new low watermark should be bigger", newWatermark > initWatermark);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java b/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java
deleted file mode 100644
index 731401e..0000000
--- a/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-public class TestOmidClientConfiguration {
-
- @Test
- public void testYamlReading() {
- OmidClientConfiguration configuration = new OmidClientConfiguration();
- Assert.assertNotNull(configuration.getConnectionString());
- Assert.assertNotNull(configuration.getConnectionType());
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/test/java/org/apache/omid/tso/client/TestMockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/org/apache/omid/tso/client/TestMockTSOClient.java b/transaction-client/src/test/java/org/apache/omid/tso/client/TestMockTSOClient.java
new file mode 100644
index 0000000..fe7d30b
--- /dev/null
+++ b/transaction-client/src/test/java/org/apache/omid/tso/client/TestMockTSOClient.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.collect.Sets;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.InMemoryCommitTable;
+import org.apache.omid.tso.util.DummyCellIdImpl;
+import org.testng.Assert;
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.AssertJUnit.assertEquals;
+
+public class TestMockTSOClient {
+
+ final static public CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
+ final static public CellId c2 = new DummyCellIdImpl(-0xfeedcafeL);
+
+ @Test(timeOut = 10000)
+ public void testConflicts() throws Exception {
+ CommitTable commitTable = new InMemoryCommitTable();
+ TSOProtocol client = new MockTSOClient(commitTable.getWriter());
+
+ long tr1 = client.getNewStartTimestamp().get();
+ long tr2 = client.getNewStartTimestamp().get();
+
+ client.commit(tr1, Sets.newHashSet(c1)).get();
+
+ try {
+ client.commit(tr2, Sets.newHashSet(c1, c2)).get();
+ Assert.fail("Shouldn't have committed");
+ } catch (ExecutionException ee) {
+ assertEquals("Should have aborted", ee.getCause().getClass(), AbortException.class);
+ }
+ }
+
+ @Test(timeOut = 10000)
+ public void testWatermarkUpdate() throws Exception {
+ CommitTable commitTable = new InMemoryCommitTable();
+ TSOProtocol client = new MockTSOClient(commitTable.getWriter());
+ CommitTable.Client commitTableClient = commitTable.getClient();
+
+ long tr1 = client.getNewStartTimestamp().get();
+ client.commit(tr1, Sets.newHashSet(c1)).get();
+
+ long initWatermark = commitTableClient.readLowWatermark().get();
+
+ long tr2 = client.getNewStartTimestamp().get();
+ client.commit(tr2, Sets.newHashSet(c1)).get();
+
+ long newWatermark = commitTableClient.readLowWatermark().get();
+ AssertJUnit.assertTrue("new low watermark should be bigger", newWatermark > initWatermark);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/test/java/org/apache/omid/tso/client/TestOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/org/apache/omid/tso/client/TestOmidClientConfiguration.java b/transaction-client/src/test/java/org/apache/omid/tso/client/TestOmidClientConfiguration.java
new file mode 100644
index 0000000..1bdb005
--- /dev/null
+++ b/transaction-client/src/test/java/org/apache/omid/tso/client/TestOmidClientConfiguration.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestOmidClientConfiguration {
+
+ @Test
+ public void testYamlReading() {
+ OmidClientConfiguration configuration = new OmidClientConfiguration();
+ Assert.assertNotNull(configuration.getConnectionString());
+ Assert.assertNotNull(configuration.getConnectionType());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/bin/omid.sh
----------------------------------------------------------------------
diff --git a/tso-server/bin/omid.sh b/tso-server/bin/omid.sh
index a559bba..66382d7 100755
--- a/tso-server/bin/omid.sh
+++ b/tso-server/bin/omid.sh
@@ -20,7 +20,7 @@ for j in ../lib/*.jar; do
done
tso() {
- exec java $JVM_FLAGS -cp $CLASSPATH com.yahoo.omid.tso.TSOServer @../conf/omid.conf $@
+ exec java $JVM_FLAGS -cp $CLASSPATH org.apache.omid.tso.TSOServer @../conf/omid.conf $@
}
tsoRelauncher() {
@@ -31,11 +31,11 @@ tsoRelauncher() {
}
createHBaseCommitTable() {
- exec java -cp $CLASSPATH com.yahoo.omid.tools.hbase.OmidTableManager commit-table $@
+ exec java -cp $CLASSPATH org.apache.omid.tools.hbase.OmidTableManager commit-table $@
}
createHBaseTimestampTable() {
- exec java -cp $CLASSPATH com.yahoo.omid.tools.hbase.OmidTableManager timestamp-table $@
+ exec java -cp $CLASSPATH org.apache.omid.tools.hbase.OmidTableManager timestamp-table $@
}
usage() {
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/maven/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/tso-server/maven/assembly/bin.xml b/tso-server/maven/assembly/bin.xml
index cd84a9d..7fe4c22 100644
--- a/tso-server/maven/assembly/bin.xml
+++ b/tso-server/maven/assembly/bin.xml
@@ -17,9 +17,9 @@
<useAllReactorProjects>true</useAllReactorProjects>
<!-- Now, select which projects to include in this module-set. -->
<includes>
- <include>com.yahoo.omid:hbase-commit-table</include>
- <include>com.yahoo.omid:codahale-metrics</include>
- <include>com.yahoo.omid:hbase-tools</include>
+ <include>org.apache.omid:hbase-commit-table</include>
+ <include>org.apache.omid:codahale-metrics</include>
+ <include>org.apache.omid:hbase-tools</include>
</includes>
<binaries>
<outputDirectory>/lib</outputDirectory>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index 2950b98..0cf0f6c 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -18,44 +18,44 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>timestamp-storage</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>metrics</artifactId>
<version>${project.version}</version>
</dependency>
<!-- The codahale-related dependency is added here just to include the jar in the final
tar.gz package assembled. -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>codahale-metrics</artifactId>
<version>${project.version}</version>
</dependency>
<!-- The hbase-related dependency is added here just to include the jar in the final tar.gz package assembled.
The hbase tools are handy to create the required Omid tables from the same script that starts the TSO server -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-tools</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>commit-table</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<classifier>tests</classifier>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>transaction-client</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -268,7 +268,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -279,7 +279,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java b/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
deleted file mode 100644
index 4fac4c5..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-/**
- * This interface defines a container for long-long mappings, for instance for holding row -> last commit information
- *
- */
-public interface Cache {
-
- public void reset();
-
- public abstract long set(long key, long value);
-
- public abstract long get(long key);
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java b/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
deleted file mode 100644
index fd9b6d7..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import java.io.FileNotFoundException;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
-import java.util.Random;
-
-public class CacheEvaluation {
-
- private final static int ENTRIES = 1000000;
- private final static int WARMUP_ROUNDS = 2;
- private final static int ROUNDS = 4;
- private final static double HOT_PERC = 1;
-
- public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
- int[] asoc = new int[]{8, 16, 32};
- for (int anAsoc : asoc) {
- PrintWriter writer = new PrintWriter(anAsoc + ".out", "UTF-8");
- new CacheEvaluation().testEntriesAge(new LongCache(ENTRIES, anAsoc), writer);
- writer.close();
- }
- PrintWriter writer = new PrintWriter("guava.out", "UTF-8");
- writer.close();
- }
-
- private void testEntriesAge(Cache cache, PrintWriter writer) {
- Random random = new Random();
-
- long seed = random.nextLong();
-
- writer.println("# Random seed: " + seed);
- random.setSeed(seed);
- int removals = 0;
- double tempStdDev = 0;
- double tempAvg = 0;
-
- int i = 0;
- int largestDeletedTimestamp = 0;
- long hotItem = random.nextLong();
-
- Runtime.getRuntime().gc();
-
- for (; i < ENTRIES * WARMUP_ROUNDS; ++i) {
- long toInsert = random.nextInt(100) < HOT_PERC ? hotItem : random.nextLong();
- long removed = cache.set(toInsert, i);
- if (removed > largestDeletedTimestamp) {
- largestDeletedTimestamp = (int) removed;
- }
- if (removed > largestDeletedTimestamp) {
- largestDeletedTimestamp = (int) removed;
- }
- if (i % ENTRIES == 0) {
- int round = i / ENTRIES + 1;
- System.err.format("Warmup [%d/%d]\n", round, WARMUP_ROUNDS);
- }
- }
-
- long time = System.nanoTime();
- for (; i < ENTRIES * (WARMUP_ROUNDS + ROUNDS); ++i) {
- long toInsert = random.nextInt(100) < HOT_PERC ? hotItem : random.nextLong();
- long removed = cache.set(toInsert, i);
- if (removed > largestDeletedTimestamp) {
- largestDeletedTimestamp = (int) removed;
- }
- int gap = i - largestDeletedTimestamp;
- removals++;
- double oldAvg = tempAvg;
- tempAvg += (gap - tempAvg) / removals;
- tempStdDev += (gap - oldAvg) * (gap - tempAvg);
- if (i % ENTRIES == 0) {
- int round = i / ENTRIES - WARMUP_ROUNDS + 1;
- System.err.format("Progress [%d/%d]\n", round, ROUNDS);
- }
- }
- long elapsed = System.nanoTime() - time;
- double elapsedSeconds = (elapsed / (double) 1000000000);
- long totalOps = ENTRIES * ROUNDS;
- writer.println("# Free mem before GC (MB) :" + (Runtime.getRuntime().freeMemory() / (double) (1024 * 1024)));
- Runtime.getRuntime().gc();
- writer.println("# Free mem (MB) :" + (Runtime.getRuntime().freeMemory() / (double) (1024 * 1024)));
- writer.println("# Elapsed (s): " + elapsedSeconds);
- writer.println("# Elapsed per 100 ops (ms): " + (elapsed / (double) totalOps / 100 / (double) 1000000));
- writer.println("# Ops per s : " + (totalOps / elapsedSeconds));
- writer.println("# Avg gap: " + (tempAvg));
- writer.println("# Std dev gap: " + Math.sqrt((tempStdDev / ENTRIES)));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java b/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
deleted file mode 100644
index 74bfa52..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Stores the mapping between a particular cell id and the commit timestamp
- * of the last transaction that changed it.
- *
- * The mapping is implemented as a long -> long mapping, using a single long [].
- * For a map of size N we create an array of size 2*N and store the keys
- * on even indexes and values on odd indexes. The rationale is that we want
- * queries to be fast and touch as least memory regions as possible.
- *
- * Each time an entry is removed, the caller updates the largestDeletedTimestamp
- * if the entry's commit timestamp is greater than this value.
- *
- * TODO: improve garbage collection, right now an entry is picked at random
- * (by hash) which could cause the eviction of a very recent timestamp
- */
-
-class CommitHashMap {
-
- private static final Logger LOG = LoggerFactory.getLogger(CommitHashMap.class);
-
- private final Cache cellIdToCommitMap;
-
- /**
- * Constructs a new, empty hashtable with a default size of 1000
- */
- public CommitHashMap() {
- this(1000);
- }
-
- /**
- * Constructs a new, empty hashtable with the specified size
- *
- * @param size
- * the initial size of the hashtable.
- * @throws IllegalArgumentException
- * if the size is less than zero.
- */
- public CommitHashMap(int size) {
- if (size < 0) {
- throw new IllegalArgumentException("Illegal size: " + size);
- }
-
- this.cellIdToCommitMap = new LongCache(size, 32);
- LOG.info("CellId -> CommitTS map created with [{}] buckets (32 elems/bucket)", size);
- }
-
- public void reset() {
- cellIdToCommitMap.reset();
- LOG.info("CellId -> CommitTS map reset");
- }
-
- public long getLatestWriteForCell(long hash) {
- return cellIdToCommitMap.get(hash);
- }
-
- public long putLatestWriteForCell(long hash, long commitTimestamp) {
- return cellIdToCommitMap.set(hash, commitTimestamp);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
deleted file mode 100644
index 7aa1e4f..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-
-import javax.inject.Singleton;
-
-public class DisruptorModule extends AbstractModule {
-
- @Override
- protected void configure() {
-
- bind(RequestProcessor.class).to(RequestProcessorImpl.class).in(Singleton.class);
- bind(PersistenceProcessor.class).to(PersistenceProcessorImpl.class).in(Singleton.class);
- bind(ReplyProcessor.class).to(ReplyProcessorImpl.class).in(Singleton.class);
- bind(RetryProcessor.class).to(RetryProcessorImpl.class).in(Singleton.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java b/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
deleted file mode 100644
index a044b04..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.lmax.disruptor.ExceptionHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class FatalExceptionHandler implements ExceptionHandler {
- private static final Logger LOG = LoggerFactory.getLogger(FatalExceptionHandler.class);
-
- Panicker panicker;
-
- FatalExceptionHandler(Panicker panicker) {
- this.panicker = panicker;
- }
-
- @Override
- public void handleEventException(Throwable ex,
- long sequence,
- Object event) {
- LOG.error("Uncaught exception throws for sequence {}, event {}",
- new Object[]{sequence, event, ex});
- panicker.panic("Uncaught exception in disruptor thread", ex);
- }
-
- @Override
- public void handleOnShutdownException(Throwable ex) {
- LOG.warn("Uncaught exception shutting down", ex);
- }
-
- @Override
- public void handleOnStartException(Throwable ex) {
- panicker.panic("Uncaught exception starting up", ex);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
deleted file mode 100644
index 5582861..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.timestamp.storage.ZKModule;
-import com.yahoo.omid.tso.LeaseManagement.LeaseManagementException;
-import org.apache.curator.framework.CuratorFramework;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-public class HALeaseManagementModule extends AbstractModule {
-
- private static final Logger LOG = LoggerFactory.getLogger(HALeaseManagementModule.class);
- private final long leasePeriodInMs;
- private final String tsoLeasePath;
- private final String currentTsoPath;
- private final String zkCluster;
- private final String zkNamespace;
-
- public HALeaseManagementModule(long leasePeriodInMs, String tsoLeasePath, String currentTsoPath,
- String zkCluster, String zkNamespace) {
-
- this.leasePeriodInMs = leasePeriodInMs;
- this.tsoLeasePath = tsoLeasePath;
- this.currentTsoPath = currentTsoPath;
- this.zkCluster = zkCluster;
- this.zkNamespace = zkNamespace;
-
- }
-
- @Override
- protected void configure() {
-
- install(new ZKModule(zkCluster, zkNamespace));
-
- }
-
- @Provides
- @Singleton
- LeaseManagement provideLeaseManager(@Named(TSO_HOST_AND_PORT_KEY) String tsoHostAndPort,
- TSOChannelHandler tsoChannelHandler,
- TSOStateManager stateManager,
- CuratorFramework zkClient,
- Panicker panicker) throws LeaseManagementException {
-
- LOG.info("Connection to HA cluster [{}]", zkClient.getState());
-
- return new LeaseManager(tsoHostAndPort,
- tsoChannelHandler,
- stateManager,
- leasePeriodInMs,
- tsoLeasePath,
- currentTsoPath,
- zkClient,
- panicker);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
deleted file mode 100644
index 98d8261..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.NullCommitTable;
-
-import javax.inject.Singleton;
-
-public class InMemoryCommitTableStorageModule extends AbstractModule {
-
- @Override
- public void configure() {
-
- bind(CommitTable.class).to(NullCommitTable.class).in(Singleton.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
deleted file mode 100644
index be1c3fc..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import com.yahoo.omid.tso.TimestampOracleImpl.InMemoryTimestampStorage;
-
-import javax.inject.Singleton;
-
-public class InMemoryTimestampStorageModule extends AbstractModule {
-
- @Override
- public void configure() {
-
- bind(TimestampStorage.class).to(InMemoryTimestampStorage.class).in(Singleton.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java b/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
deleted file mode 100644
index 8736f8d..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-public interface LeaseManagement {
-
- class LeaseManagementException extends Exception {
-
- private static final long serialVersionUID = -2061376444591776881L;
-
- LeaseManagementException(String msg) {
- super(msg);
- }
-
-
- LeaseManagementException(String msg, Exception e) {
- super(msg, e);
- }
-
- }
-
- /**
- * Allows to start the service implementing the lease management
- */
- void startService() throws LeaseManagementException;
-
- /**
- * Allows to stop the service implementing the lease management
- */
- void stopService() throws LeaseManagementException;
-
- /**
- * Check if the instance is still is under the lease period
- */
- boolean stillInLeasePeriod();
-
-}
[17/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java
deleted file mode 100644
index 03dbbd7..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.proto.TSOProto.Response;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ExecutionException;
-
-/**
- * Communication endpoint for TSO clients.
- */
-public class TSOClientOneShot {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOClientOneShot.class);
-
- private final String host;
- private final int port;
-
- public TSOClientOneShot(String host, int port) {
-
- this.host = host;
- this.port = port;
-
- }
-
- public TSOProto.Response makeRequest(TSOProto.Request request)
- throws InterruptedException, ExecutionException {
- TSOClientRaw raw = new TSOClientRaw(host, port);
-
- // do handshake
- TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
- handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
- raw.write(TSOProto.Request.newBuilder()
- .setHandshakeRequest(handshake.build()).build());
- Response response = raw.getResponse().get();
- assert (response.getHandshakeResponse().getClientCompatible());
-
- raw.write(request);
- response = raw.getResponse().get();
-
- raw.close();
- return response;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java
deleted file mode 100644
index 9a68c39..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.proto.TSOProto.Response;
-import org.jboss.netty.bootstrap.ClientBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
-import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
-import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- * Raw client for communicating with tso server directly with protobuf messages
- */
-public class TSOClientRaw {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOClientRaw.class);
-
- private final BlockingQueue<SettableFuture<Response>> responseQueue
- = new ArrayBlockingQueue<SettableFuture<Response>>(5);
- private final Channel channel;
-
- public TSOClientRaw(String host, int port) throws InterruptedException, ExecutionException {
- // Start client with Nb of active threads = 3 as maximum.
- ChannelFactory factory = new NioClientSocketChannelFactory(
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), 3);
- // Create the bootstrap
- ClientBootstrap bootstrap = new ClientBootstrap(factory);
-
- InetSocketAddress addr = new InetSocketAddress(host, port);
-
- ChannelPipeline pipeline = bootstrap.getPipeline();
- pipeline.addLast("lengthbaseddecoder",
- new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
- pipeline.addLast("protobufdecoder",
- new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
- pipeline.addLast("protobufencoder", new ProtobufEncoder());
-
- Handler handler = new Handler();
- pipeline.addLast("handler", handler);
-
- bootstrap.setOption("tcpNoDelay", true);
- bootstrap.setOption("keepAlive", true);
- bootstrap.setOption("reuseAddress", true);
- bootstrap.setOption("connectTimeoutMillis", 100);
-
- ChannelFuture channelFuture = bootstrap.connect(addr).await();
- channel = channelFuture.getChannel();
- }
-
- public void write(TSOProto.Request request) {
- channel.write(request);
- }
-
- public Future<Response> getResponse() throws InterruptedException {
- SettableFuture<Response> future = SettableFuture.<Response>create();
- responseQueue.put(future);
- return future;
- }
-
- public void close() throws InterruptedException {
- responseQueue.put(SettableFuture.<Response>create());
- channel.close();
- }
-
- private class Handler extends SimpleChannelHandler {
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
- LOG.info("Message received", e);
- if (e.getMessage() instanceof Response) {
- Response resp = (Response) e.getMessage();
- try {
- SettableFuture<Response> future = responseQueue.take();
- future.set(resp);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted in handler", ie);
- }
- } else {
- LOG.warn("Received unknown message", e.getMessage());
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
- LOG.info("Exception received", e.getCause());
- try {
- SettableFuture<Response> future = responseQueue.take();
- future.setException(e.getCause());
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted handling exception", ie);
- }
- }
-
- @Override
- public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e)
- throws Exception {
- LOG.info("Disconnected");
- try {
- SettableFuture<Response> future = responseQueue.take();
- future.setException(new ConnectionException());
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted handling exception", ie);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
deleted file mode 100644
index c89d05e..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.collect.Sets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-import static org.testng.AssertJUnit.assertNotNull;
-
-public class TestIntegrationOfTSOClientServerBasicFunctionality {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestIntegrationOfTSOClientServerBasicFunctionality.class);
-
- private static final String TSO_SERVER_HOST = "localhost";
- private int tsoServerPortForTest;
-
- // Cells for tests
- private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
- private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
-
- // Required infrastructure for TSO tsoClient-server integration testing
- private TSOServer tsoServer;
- private TSOClient tsoClient;
- private TSOClient justAnotherTSOClient;
- private CommitTable.Client commitTableClient;
-
- @BeforeClass
- public void setup() throws Exception {
-
- tsoServerPortForTest = TestUtils.getFreeLocalPort();
-
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setMaxItems(1000);
- tsoConfig.setPort(tsoServerPortForTest);
- Module tsoServerMockModule = new TSOMockModule(tsoConfig);
- Injector injector = Guice.createInjector(tsoServerMockModule);
-
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- commitTableClient = commitTable.getClient();
-
- LOG.info("==================================================================================================");
- LOG.info("======================================= Init TSO Server ==========================================");
- LOG.info("==================================================================================================");
-
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_SERVER_HOST, tsoServerPortForTest, 100);
-
- LOG.info("==================================================================================================");
- LOG.info("===================================== TSO Server Initialized =====================================");
- LOG.info("==================================================================================================");
-
- LOG.info("==================================================================================================");
- LOG.info("======================================= Setup TSO Clients ========================================");
- LOG.info("==================================================================================================");
-
- // Configure direct connection to the server
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + tsoServerPortForTest);
-
- tsoClient = TSOClient.newInstance(tsoClientConf);
- justAnotherTSOClient = TSOClient.newInstance(tsoClientConf);
-
- LOG.info("==================================================================================================");
- LOG.info("===================================== TSO Clients Initialized ====================================");
- LOG.info("==================================================================================================");
-
- Thread.currentThread().setName("Test Thread");
-
- }
-
- @AfterClass
- public void tearDown() throws Exception {
-
- tsoClient.close().get();
-
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, tsoServerPortForTest, 1000);
-
- }
-
- @Test(timeOut = 30_000)
- public void testTimestampsOrderingGrowMonotonically() throws Exception {
- long referenceTimestamp;
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- referenceTimestamp = startTsTx1;
-
- long startTsTx2 = tsoClient.getNewStartTimestamp().get();
- assertEquals(startTsTx2, ++referenceTimestamp, "Should grow monotonically");
- assertTrue(startTsTx2 > startTsTx1, "Two timestamps obtained consecutively should grow");
-
- long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1)).get();
- assertEquals(commitTsTx2, ++referenceTimestamp, "Should grow monotonically");
-
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c2)).get();
- assertEquals(commitTsTx1, ++referenceTimestamp, "Should grow monotonically");
-
- long startTsTx3 = tsoClient.getNewStartTimestamp().get();
- assertEquals(startTsTx3, ++referenceTimestamp, "Should grow monotonically");
- }
-
- @Test(timeOut = 30_000)
- public void testSimpleTransactionWithNoWriteSetCanCommit() throws Exception {
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.<CellId>newHashSet()).get();
- assertTrue(commitTsTx1 > startTsTx1);
- }
-
- @Test(timeOut = 30_000)
- public void testTransactionWithMassiveWriteSetCanCommit() throws Exception {
- long startTs = tsoClient.getNewStartTimestamp().get();
-
- Set<CellId> cells = new HashSet<>();
- for (int i = 0; i < 1_000_000; i++) {
- cells.add(new DummyCellIdImpl(i));
- }
-
- long commitTs = tsoClient.commit(startTs, cells).get();
- assertTrue(commitTs > startTs, "Commit TS should be higher than Start TS");
- }
-
- @Test(timeOut = 30_000)
- public void testMultipleSerialCommitsDoNotConflict() throws Exception {
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
- assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be greater than Start TS");
-
- long startTsTx2 = tsoClient.getNewStartTimestamp().get();
- assertTrue(startTsTx2 > commitTsTx1, "TS should grow monotonically");
-
- long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
- assertTrue(commitTsTx2 > startTsTx2, "Commit TS must be greater than Start TS");
-
- long startTsTx3 = tsoClient.getNewStartTimestamp().get();
- long commitTsTx3 = tsoClient.commit(startTsTx3, Sets.newHashSet(c2)).get();
- assertTrue(commitTsTx3 > startTsTx3, "Commit TS must be greater than Start TS");
- }
-
- @Test(timeOut = 30_000)
- public void testCommitWritesToCommitTable() throws Exception {
- long startTsForTx1 = tsoClient.getNewStartTimestamp().get();
- long startTsForTx2 = tsoClient.getNewStartTimestamp().get();
- assertTrue(startTsForTx2 > startTsForTx1, "Start TS should grow");
-
- assertFalse(commitTableClient.getCommitTimestamp(startTsForTx1).get().isPresent(),
- "Commit TS for Tx1 shouldn't appear in Commit Table");
-
- long commitTsForTx1 = tsoClient.commit(startTsForTx1, Sets.newHashSet(c1)).get();
- assertTrue(commitTsForTx1 > startTsForTx1, "Commit TS should be higher than Start TS for the same tx");
-
- Long commitTs1InCommitTable = commitTableClient.getCommitTimestamp(startTsForTx1).get().get().getValue();
- assertNotNull("Tx is committed, should return as such from Commit Table", commitTs1InCommitTable);
- assertEquals(commitTsForTx1, (long) commitTs1InCommitTable,
- "getCommitTimestamp() & commit() should report same Commit TS value for same tx");
- assertTrue(commitTs1InCommitTable > startTsForTx2, "Commit TS should be higher than tx's Start TS");
- }
-
- @Test(timeOut = 30_000)
- public void testTwoConcurrentTxWithOverlappingWritesetsHaveConflicts() throws Exception {
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- long startTsTx2 = tsoClient.getNewStartTimestamp().get();
- assertTrue(startTsTx2 > startTsTx1, "Second TX should have higher TS");
-
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
- assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be higher than Start TS for the same tx");
-
- try {
- tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
- Assert.fail("Second TX should fail on commit");
- } catch (ExecutionException ee) {
- assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
- }
- }
-
- @Test(timeOut = 30_000)
- public void testConflictsAndMonotonicallyTimestampGrowthWithTwoDifferentTSOClients() throws Exception {
- long startTsTx1Client1 = tsoClient.getNewStartTimestamp().get();
- long startTsTx2Client1 = tsoClient.getNewStartTimestamp().get();
- long startTsTx3Client1 = tsoClient.getNewStartTimestamp().get();
-
- tsoClient.commit(startTsTx1Client1, Sets.newHashSet(c1)).get();
- try {
- tsoClient.commit(startTsTx3Client1, Sets.newHashSet(c1, c2)).get();
- Assert.fail("Second commit should fail as conflicts with the previous concurrent one");
- } catch (ExecutionException ee) {
- assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
- }
- long startTsTx4Client2 = justAnotherTSOClient.getNewStartTimestamp().get();
-
- assertFalse(commitTableClient.getCommitTimestamp(startTsTx3Client1).get().isPresent(), "Tx3 didn't commit");
- long commitTSTx1 = commitTableClient.getCommitTimestamp(startTsTx1Client1).get().get().getValue();
- assertTrue(commitTSTx1 > startTsTx2Client1, "Tx1 committed after Tx2 started");
- assertTrue(commitTSTx1 < startTsTx4Client2, "Tx1 committed before Tx4 started on the other TSO Client");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java
deleted file mode 100644
index 581e5ce..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.tso.HALeaseManagementModule;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.VoidLeaseManagementModule;
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.test.TestingServer;
-import org.apache.curator.utils.CloseableUtils;
-import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-public class TestTSOClientConnectionToTSO {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientConnectionToTSO.class);
-
- // Constants and variables for component connectivity
- private static final String TSO_HOST = "localhost";
- private static final String CURRENT_TSO_PATH = "/current_tso_path";
- private static final String TSO_LEASE_PATH = "/tso_lease_path";
-
- private int tsoPortForTest;
- private String zkClusterForTest;
-
- private Injector injector = null;
-
- private TestingServer zkServer;
-
- private CuratorFramework zkClient;
- private TSOServer tsoServer;
-
- @BeforeMethod
- public void beforeMethod() throws Exception {
-
- tsoPortForTest = TestUtils.getFreeLocalPort();
-
- int zkPortForTest = TestUtils.getFreeLocalPort();
- zkClusterForTest = TSO_HOST + ":" + zkPortForTest;
- LOG.info("Starting ZK Server in port {}", zkPortForTest);
- zkServer = TestUtils.provideTestingZKServer(zkPortForTest);
- LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
-
- zkClient = TestUtils.provideConnectedZKClient(zkClusterForTest);
-
- Stat stat;
- try {
- zkClient.delete().forPath(CURRENT_TSO_PATH);
- stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
- assertNull(stat, CURRENT_TSO_PATH + " should not exist");
- } catch (NoNodeException e) {
- LOG.info("{} ZNode did not exist", CURRENT_TSO_PATH);
- }
-
- }
-
- @AfterMethod
- public void afterMethod() {
-
- zkClient.close();
-
- CloseableUtils.closeQuietly(zkServer);
- zkServer = null;
- LOG.info("ZK Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testUnsuccessfulConnectionToTSO() throws Exception {
-
- // When no HA node for TSOServer is found & no host:port config exists
- // we should get an exception when getting the client
- try {
- TSOClient.newInstance(new OmidClientConfiguration());
- } catch (IllegalArgumentException e) {
- // Expected
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testSuccessfulConnectionToTSOWithHostAndPort() throws Exception {
-
- // Launch a TSO WITHOUT publishing the address in HA...
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setMaxItems(1000);
- tsoConfig.setPort(tsoPortForTest);
- tsoConfig.setLeaseModule(new VoidLeaseManagementModule());
- injector = Guice.createInjector(new TSOMockModule(tsoConfig));
- LOG.info("Starting TSO");
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading TSO");
-
- // When no HA node for TSOServer is found we should get a connection
- // to the TSO through the host:port configured...
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString("localhost:" + tsoPortForTest);
- tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // ... so we should get responses from the methods
- Long startTS = tsoClient.getNewStartTimestamp().get();
- LOG.info("Start TS {} ", startTS);
- assertEquals(startTS.longValue(), 1);
-
- // Close the tsoClient connection and stop the TSO Server
- tsoClient.close().get();
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("TSO Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testSuccessfulConnectionToTSOThroughZK() throws Exception {
-
- // Launch a TSO publishing the address in HA...
- TSOServerConfig config = new TSOServerConfig();
- config.setMaxItems(1000);
- config.setPort(tsoPortForTest);
- config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
- injector = Guice.createInjector(new TSOMockModule(config));
- LOG.info("Starting TSO");
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading TSO");
-
- waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
-
- // When a HA node for TSOServer is found we should get a connection
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionType(OmidClientConfiguration.ConnType.HA);
- tsoClientConf.setConnectionString(zkClusterForTest);
- tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // ... so we should get responses from the methods
- Long startTS = tsoClient.getNewStartTimestamp().get();
- LOG.info("Start TS {} ", startTS);
- assertEquals(startTS.longValue(), 1);
-
- // Close the tsoClient connection and stop the TSO Server
- tsoClient.close().get();
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("TSO Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testSuccessOfTSOClientReconnectionsToARestartedTSOWithZKPublishing() throws Exception {
-
- // Start a TSO with HA...
- TSOServerConfig config = new TSOServerConfig();
- config.setMaxItems(1000);
- config.setPort(tsoPortForTest);
- config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
- injector = Guice.createInjector(new TSOMockModule(config));
- LOG.info("Starting Initial TSO");
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading TSO");
-
- waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
-
- // Then create the TSO Client under test...
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionType(OmidClientConfiguration.ConnType.HA);
- tsoClientConf.setConnectionString(zkClusterForTest);
- tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // ... and check that initially we get responses from the methods
- Long startTS = tsoClient.getNewStartTimestamp().get();
- LOG.info("Start TS {} ", startTS);
- assertEquals(startTS.longValue(), 1);
-
- // Then stop the server...
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("Initial TSO Server Stopped");
-
- Thread.sleep(1500); // ...allow the client to receive disconnection event...
- // ... and check that we get a conn exception when trying to access the client
- try {
- startTS = tsoClient.getNewStartTimestamp().get();
- fail();
- } catch (ExecutionException e) {
- LOG.info("Exception expected");
- // Internal accessor to fsm to do the required checkings
- FsmImpl fsm = (FsmImpl) tsoClient.fsm;
- assertEquals(e.getCause().getClass(), ConnectionException.class);
- assertTrue(fsm.getState().getClass().equals(TSOClient.ConnectionFailedState.class)
- ||
- fsm.getState().getClass().equals(TSOClient.DisconnectedState.class));
- }
-
- // After that, simulate that a new TSO has been launched...
- Injector newInjector = Guice.createInjector(new TSOMockModule(config));
- LOG.info("Re-Starting again the TSO");
- tsoServer = newInjector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading restarted TSO");
-
- // Finally re-check that, eventually, we can get a new value from the new TSO...
- boolean reconnectionActive = false;
- while (!reconnectionActive) {
- try {
- startTS = tsoClient.getNewStartTimestamp().get();
- reconnectionActive = true;
- } catch (ExecutionException e) {
- // Expected
- }
- }
- assertNotNull(startTS);
-
- // ...and stop the server
- tsoServer.stopAndWait();
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("Restarted TSO Server Stopped");
- }
-
- private void waitTillTsoRegisters(CuratorFramework zkClient) throws Exception {
- while (true) {
- try {
- Stat stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
- if (stat == null) {
- continue;
- }
- LOG.info("TSO registered in HA with path {}={}", CURRENT_TSO_PATH, stat.toString());
- if (stat.toString().length() == 0) {
- continue;
- }
- return;
- } catch (Exception e) {
- LOG.debug("TSO still has not registered yet, sleeping...", e);
- Thread.sleep(500);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
deleted file mode 100644
index 0e15261..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.collect.Sets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.tso.PausableTimestampOracle;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.TimestampOracle;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-public class TestTSOClientRequestAndResponseBehaviours {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientRequestAndResponseBehaviours.class);
-
- private static final String TSO_SERVER_HOST = "localhost";
- private static final int TSO_SERVER_PORT = 1234;
-
- private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
- private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
-
- private final static Set<CellId> testWriteSet = Sets.newHashSet(c1, c2);
-
- private OmidClientConfiguration tsoClientConf;
-
- // Required infrastructure for TSOClient test
- private TSOServer tsoServer;
- private PausableTimestampOracle pausableTSOracle;
- private CommitTable commitTable;
-
- @BeforeClass
- public void setup() throws Exception {
-
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setMaxItems(1000);
- tsoConfig.setPort(TSO_SERVER_PORT);
- Module tsoServerMockModule = new TSOMockModule(tsoConfig);
- Injector injector = Guice.createInjector(tsoServerMockModule);
-
- LOG.info("==================================================================================================");
- LOG.info("======================================= Init TSO Server ==========================================");
- LOG.info("==================================================================================================");
-
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 100);
-
- LOG.info("==================================================================================================");
- LOG.info("===================================== TSO Server Initialized =====================================");
- LOG.info("==================================================================================================");
-
- pausableTSOracle = (PausableTimestampOracle) injector.getInstance(TimestampOracle.class);
- commitTable = injector.getInstance(CommitTable.class);
-
- }
-
- @AfterClass
- public void tearDown() throws Exception {
-
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 1000);
-
- }
-
- @BeforeMethod
- public void beforeMethod() {
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
-
- this.tsoClientConf = tsoClientConf;
-
- }
-
- @AfterMethod
- public void afterMethod() {
-
- pausableTSOracle.resume();
-
- }
-
- /**
- * Test to ensure TSOClient timeouts are cancelled.
- * At some point a bug was detected because the TSOClient timeouts were not cancelled, and as timestamp requests
- * had no way to be correlated to timestamp responses, random requests were just timed out after a certain time.
- * We send a lot of timestamp requests, and wait for them to complete.
- * Ensure that the next request doesn't get hit by the timeouts of the previous
- * requests. (i.e. make sure we cancel timeouts)
- */
- @Test(timeOut = 30_000)
- public void testTimeoutsAreCancelled() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- int requestTimeoutInMs = 500;
- int requestMaxRetries = 5;
- LOG.info("Request timeout {} ms; Max retries {}", requestTimeoutInMs, requestMaxRetries);
- Future<Long> f = null;
- for (int i = 0; i < (requestMaxRetries * 10); i++) {
- f = client.getNewStartTimestamp();
- }
- if (f != null) {
- f.get();
- }
- pausableTSOracle.pause();
- long msToSleep = ((long) (requestTimeoutInMs * 0.75));
- LOG.info("Sleeping for {} ms", msToSleep);
- TimeUnit.MILLISECONDS.sleep(msToSleep);
- f = client.getNewStartTimestamp();
- msToSleep = ((long) (requestTimeoutInMs * 0.9));
- LOG.info("Sleeping for {} ms", msToSleep);
- TimeUnit.MILLISECONDS.sleep(msToSleep);
- LOG.info("Resuming");
- pausableTSOracle.resume();
- f.get();
-
- }
-
- @Test(timeOut = 30_000)
- public void testCommitGetsServiceUnavailableExceptionWhenCommunicationFails() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestMaxRetries(0);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- List<Long> startTimestamps = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- startTimestamps.add(client.getNewStartTimestamp().get());
- }
-
- pausableTSOracle.pause();
-
- List<Future<Long>> futures = new ArrayList<>();
- for (long s : startTimestamps) {
- futures.add(client.commit(s, Sets.<CellId>newHashSet()));
- }
- TSOClientAccessor.closeChannel(client);
-
- for (Future<Long> f : futures) {
- try {
- f.get();
- fail("Shouldn't be able to complete");
- } catch (ExecutionException ee) {
- assertTrue(ee.getCause() instanceof ServiceUnavailableException,
- "Should be a service unavailable exception");
- }
- }
- }
-
- /**
- * Test that if a client tries to make a request without handshaking, it will be disconnected.
- */
- @Test(timeOut = 30_000)
- public void testHandshakeBetweenOldClientAndCurrentServer() throws Exception {
-
- TSOClientRaw raw = new TSOClientRaw(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- TSOProto.Request request = TSOProto.Request.newBuilder()
- .setTimestampRequest(TSOProto.TimestampRequest.newBuilder().build())
- .build();
- raw.write(request);
- try {
- raw.getResponse().get();
- fail("Channel should be closed");
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), ConnectionException.class, "Should be channel closed exception");
- }
- raw.close();
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Test duplicate commits
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * This tests the case where messages arrive at the TSO out of order. This can happen in the case
- * the channel get dropped and the retry is done in a new channel. However, the TSO will respond with
- * aborted to the original message because the retry was already committed and it would be prohibitively
- * expensive to check all non-retry requests to see if they are already committed. For this reason
- * a client must ensure that if it is sending a retry due to a socket error, the previous channel
- * must be entirely closed so that it will not actually receive the abort response. TCP guarantees
- * that this doesn't happen in non-socket error cases.
- *
- */
- @Test(timeOut = 30_000)
- public void testOutOfOrderMessages() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long ts1 = client.getNewStartTimestamp().get();
-
- TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
- TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
- assertFalse(response1.getCommitResponse().getAborted(), "Retry Transaction should commit");
- assertTrue(response2.getCommitResponse().getAborted(), "Transaction should abort");
- }
-
- @Test(timeOut = 30_000)
- public void testDuplicateCommitAborting() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long ts1 = client.getNewStartTimestamp().get();
- long ts2 = client.getNewStartTimestamp().get();
- client.commit(ts2, testWriteSet).get();
-
- TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
- TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
- assertTrue(response1.getCommitResponse().getAborted(), "Transaction should abort");
- assertTrue(response2.getCommitResponse().getAborted(), "Retry commit should abort");
- }
-
- @Test(timeOut = 30_000)
- public void testDuplicateCommit() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long ts1 = client.getNewStartTimestamp().get();
-
- TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
- TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
- assertEquals(response2.getCommitResponse().getCommitTimestamp(),
- response1.getCommitResponse().getCommitTimestamp(),
- "Commit timestamp should be the same");
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Test TSOClient retry behaviour
- // ----------------------------------------------------------------------------------------------------------------
-
- @Test(timeOut = 30_000)
- public void testCommitCanSucceedWhenChannelDisconnected() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
-
- long ts1 = client.getNewStartTimestamp().get();
- pausableTSOracle.pause();
- TSOFuture<Long> future = client.commit(ts1, testWriteSet);
- TSOClientAccessor.closeChannel(client);
- pausableTSOracle.resume();
- future.get();
-
- }
-
- @Test(timeOut = 30_000)
- public void testCommitCanSucceedWithMultipleTimeouts() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestTimeoutInMs(100);
- testTSOClientConf.setRequestMaxRetries(10000);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- long ts1 = client.getNewStartTimestamp().get();
- pausableTSOracle.pause();
- TSOFuture<Long> future = client.commit(ts1, testWriteSet);
- TimeUnit.SECONDS.sleep(1);
- pausableTSOracle.resume();
- future.get();
- }
-
- @Test(timeOut = 30_000)
- public void testCommitFailWhenTSOIsDown() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestTimeoutInMs(100);
- testTSOClientConf.setRequestMaxRetries(10);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- long ts1 = client.getNewStartTimestamp().get();
- pausableTSOracle.pause();
- TSOFuture<Long> future = client.commit(ts1, testWriteSet);
- try {
- future.get();
- } catch (ExecutionException e) {
- assertEquals(e.getCause().getClass(), ServiceUnavailableException.class,
- "Should be a ServiceUnavailableExeption");
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testTimestampRequestSucceedWithMultipleTimeouts() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestTimeoutInMs(100);
- testTSOClientConf.setRequestMaxRetries(10000);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- pausableTSOracle.pause();
- Future<Long> future = client.getNewStartTimestamp();
- TimeUnit.SECONDS.sleep(1);
- pausableTSOracle.resume();
- future.get();
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // The next 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
- // (They exercise the communication protocol) TODO Remove???
- // ----------------------------------------------------------------------------------------------------------------
- @Test
- public void testCommitTimestampPresentInCommitTableReturnsCommit() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long tx1ST = client.getNewStartTimestamp().get();
-
- clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- assertFalse(response.getCommitResponse().getAborted(), "Transaction should be committed");
- assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
- assertEquals(response.getCommitResponse().getCommitTimestamp(), tx1ST + 1);
- }
-
- @Test
- public void testInvalidCommitTimestampPresentInCommitTableReturnsAbort() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long tx1ST = client.getNewStartTimestamp().get();
- // Invalidate the transaction
- commitTable.getClient().tryInvalidateTransaction(tx1ST);
-
- clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- assertTrue(response.getCommitResponse().getAborted(), "Transaction should be aborted");
- assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
- assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
- }
-
- @Test
- public void testCommitTimestampNotPresentInCommitTableReturnsAnAbort() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long tx1ST = client.getNewStartTimestamp().get();
-
- clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
-
- // Simulate remove entry from the commit table before exercise retry
- commitTable.getClient().completeTransaction(tx1ST);
-
- TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- assertTrue(response.getCommitResponse().getAborted(), "Transaction should abort");
- assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
- assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
- }
- // ----------------------------------------------------------------------------------------------------------------
- // The previous 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
- // (They exercise the communication protocol) TODO Remove???
- // ----------------------------------------------------------------------------------------------------------------
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private TSOProto.Request createRetryCommitRequest(long ts) {
- return createCommitRequest(ts, true, testWriteSet);
- }
-
- private TSOProto.Request createCommitRequest(long ts, boolean retry, Set<CellId> writeSet) {
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
- commitBuilder.setStartTimestamp(ts);
- commitBuilder.setIsRetry(retry);
- for (CellId cell : writeSet) {
- commitBuilder.addCellId(cell.getCellId());
- }
- return builder.setCommitRequest(commitBuilder.build()).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java
deleted file mode 100644
index 3c2aa1e..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.yahoo.omid.tso.ProgrammableTSOServer;
-import com.yahoo.omid.tso.ProgrammableTSOServer.AbortResponse;
-import com.yahoo.omid.tso.ProgrammableTSOServer.CommitResponse;
-import com.yahoo.omid.tso.ProgrammableTSOServer.TimestampResponse;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.Assert.assertEquals;
-
-public class TestTSOClientResponseHandling {
-
- private static final int TSO_PORT = 4321;
- private static final long START_TS = 1L;
- private static final long COMMIT_TS = 2L;
-
- private ProgrammableTSOServer tsoServer = new ProgrammableTSOServer(TSO_PORT);
- // Client under test
- private TSOClient tsoClient;
-
- @BeforeClass
- public void configureAndCreateClient() throws IOException, InterruptedException {
-
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString("localhost:" + TSO_PORT);
- tsoClient = TSOClient.newInstance(tsoClientConf);
- }
-
- @BeforeMethod
- public void reset() {
- tsoServer.cleanResponses();
- }
-
- @Test
- public void testTimestampRequestReceivingASuccessfulResponse() throws Exception {
- // test request timestamp response returns a timestamp
-
- // Program the TSO to return an ad-hoc Timestamp response
- tsoServer.queueResponse(new TimestampResponse(START_TS));
-
- long startTS = tsoClient.getNewStartTimestamp().get();
- assertEquals(startTS, START_TS);
- }
-
- @Test
- public void testCommitRequestReceivingAnAbortResponse() throws Exception {
- // test commit request which is aborted on the server side
- // (e.g. due to conflicts with other transaction) throws an
- // execution exception with an AbortException as a cause
-
- // Program the TSO to return an Abort response
- tsoServer.queueResponse(new AbortResponse(START_TS));
-
- try {
- tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), AbortException.class);
- }
- }
-
- @Test
- public void testCommitRequestReceivingASuccessfulResponse() throws Exception {
- // test commit request which is successfully committed on the server
- // side returns a commit timestamp
-
- // Program the TSO to return an Commit response (with no required heuristic actions)
- tsoServer.queueResponse(new CommitResponse(false, START_TS, COMMIT_TS));
-
- long commitTS = tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
- assertEquals(commitTS, COMMIT_TS);
- }
-
- @Test
- public void testCommitRequestReceivingAHeuristicResponse() throws Exception {
- // test commit request which needs heuristic actions from the client
- // throws an execution exception with a NewTSOException as a cause
-
- // Program the TSO to return an Commit response requiring heuristic actions
- tsoServer.queueResponse(new CommitResponse(true, START_TS, COMMIT_TS));
- try {
- tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), NewTSOException.class);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java
deleted file mode 100644
index 70dec4e..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-import org.slf4j.Logger;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.collect.Sets.newHashSet;
-import static org.slf4j.LoggerFactory.getLogger;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-/**
- * Test the behavior of requests on a TSOClient component that is not connected to a TSO server.
- */
-public class TestUnconnectedTSOClient {
-
- private static final Logger LOG = getLogger(TestUnconnectedTSOClient.class);
-
- private static final int TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST = 2;
-
- @Test(timeOut = 30_000) // 30 secs
- public void testRequestsDoneOnAnUnconnectedTSOClientAlwaysReturn() throws Exception {
-
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString("localhost:12345");
- tsoClientConf.setReconnectionDelayInSecs(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST);
-
- // Component under test
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // Internal accessor to fsm
- FsmImpl fsm = (FsmImpl) tsoClient.fsm;
-
- assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
-
- // Test requests to the 3 relevant methods in TSO client
-
- try {
- tsoClient.getNewStartTimestamp().get();
- fail();
- } catch (ExecutionException e) {
- LOG.info("Exception expected");
- assertEquals(e.getCause().getClass(), ConnectionException.class);
- TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
- assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
- }
-
- try {
- tsoClient.commit(1, newHashSet(new DummyCellIdImpl(0xdeadbeefL))).get();
- fail();
- } catch (ExecutionException e) {
- LOG.info("Exception expected");
- assertEquals(e.getCause().getClass(), ConnectionException.class);
- TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
- assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
- }
-
- tsoClient.close().get();
- LOG.info("No exception expected");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/TestUtils.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/TestUtils.java b/tso-server/src/test/java/org/apache/omid/TestUtils.java
new file mode 100644
index 0000000..0d353c4
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/TestUtils.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.TestingServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.Socket;
+
+/**
+ * This class contains functionality that is useful for the Omid tests.
+ */
+public class TestUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestUtils.class);
+
+ private static final int DEFAULT_ZK_PORT = 2181;
+
+ public static int getFreeLocalPort() throws IOException {
+
+ try (ServerSocket socket = new ServerSocket(0)) {
+ socket.setReuseAddress(true);
+ return socket.getLocalPort();
+ }
+
+ }
+
+ public static TestingServer provideTestingZKServer(int port) throws Exception {
+
+ return new TestingServer(port);
+
+ }
+
+ public static TestingServer provideTestingZKServer() throws Exception {
+
+ return provideTestingZKServer(DEFAULT_ZK_PORT);
+
+ }
+
+ public static CuratorFramework provideConnectedZKClient(String zkCluster) throws Exception {
+
+ LOG.info("Creating Zookeeper Client connecting to {}", zkCluster);
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ CuratorFramework zkClient = CuratorFrameworkFactory.builder().namespace("omid")
+ .connectString(zkCluster).retryPolicy(retryPolicy).build();
+
+ LOG.info("Connecting to ZK cluster {}", zkClient.getState());
+ zkClient.start();
+ zkClient.blockUntilConnected();
+ LOG.info("Connection to ZK cluster {}", zkClient.getState());
+
+ return zkClient;
+ }
+
+ public static void waitForSocketListening(String host, int port, int sleepTimeMillis)
+ throws IOException, InterruptedException {
+ while (true) {
+ Socket sock = null;
+ try {
+ sock = new Socket(host, port);
+ } catch (IOException e) {
+ // ignore as this is expected
+ Thread.sleep(sleepTimeMillis);
+ continue;
+ } finally {
+ IOUtils.closeQuietly(sock);
+ }
+ LOG.info("Host " + host + ":" + port + " is up...");
+ break;
+ }
+ }
+
+ public static void waitForSocketNotListening(String host, int port, int sleepTimeMillis)
+ throws IOException, InterruptedException {
+ while (true) {
+ Socket sock = null;
+ try {
+ sock = new Socket(host, port);
+ } catch (IOException e) {
+ // ignore as this is expected
+ break;
+ } finally {
+ IOUtils.closeQuietly(sock);
+ }
+ Thread.sleep(sleepTimeMillis);
+ LOG.info("Host " + host + ":" + port + " is still up...");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/PausableLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/PausableLeaseManager.java b/tso-server/src/test/java/org/apache/omid/tso/PausableLeaseManager.java
new file mode 100644
index 0000000..385051f
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/PausableLeaseManager.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PausableLeaseManager extends LeaseManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PausableLeaseManager.class);
+
+ private volatile boolean pausedInTryToGetInitialLeasePeriod = false;
+ private volatile boolean pausedInTryToRenewLeasePeriod = false;
+ private volatile boolean pausedInStillInLeasePeriod = false;
+
+ public PausableLeaseManager(String id,
+ TSOChannelHandler tsoChannelHandler,
+ TSOStateManager stateManager,
+ long leasePeriodInMs,
+ String tsoLeasePath,
+ String currentTSOPath,
+ CuratorFramework zkClient,
+ Panicker panicker) {
+ super(id, tsoChannelHandler, stateManager, leasePeriodInMs, tsoLeasePath, currentTSOPath, zkClient, panicker);
+ }
+
+ @Override
+ public void tryToGetInitialLeasePeriod() throws Exception {
+ while (pausedInTryToGetInitialLeasePeriod) {
+ synchronized (this) {
+ try {
+ LOG.info("{} paused in tryToGetInitialLeasePeriod()", this);
+ this.wait();
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted whilst paused");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ super.tryToGetInitialLeasePeriod();
+ }
+
+ @Override
+ public void tryToRenewLeasePeriod() throws Exception {
+ while (pausedInTryToRenewLeasePeriod) {
+ synchronized (this) {
+ try {
+ LOG.info("{} paused in tryToRenewLeasePeriod()", this);
+ this.wait();
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted whilst paused");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ super.tryToRenewLeasePeriod();
+ }
+
+ @Override
+ public boolean stillInLeasePeriod() {
+ while (pausedInStillInLeasePeriod) {
+ synchronized (this) {
+ try {
+ LOG.info("{} paused in stillInLeasePeriod()", this);
+ this.wait();
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted whilst paused");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ return super.stillInLeasePeriod();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper Methods to pause functionality
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public synchronized void pausedInTryToGetInitialLeasePeriod() {
+ pausedInTryToGetInitialLeasePeriod = true;
+ this.notifyAll();
+ }
+
+ public synchronized void pausedInTryToRenewLeasePeriod() {
+ pausedInTryToRenewLeasePeriod = true;
+ this.notifyAll();
+ }
+
+ public synchronized void pausedInStillInLeasePeriod() {
+ pausedInStillInLeasePeriod = true;
+ this.notifyAll();
+ }
+
+ public synchronized void resume() {
+ pausedInTryToGetInitialLeasePeriod = false;
+ pausedInTryToRenewLeasePeriod = false;
+ pausedInStillInLeasePeriod = false;
+ this.notifyAll();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/ProgrammableTSOServer.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/ProgrammableTSOServer.java b/tso-server/src/test/java/org/apache/omid/tso/ProgrammableTSOServer.java
new file mode 100644
index 0000000..016938a
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/ProgrammableTSOServer.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.proto.TSOProto;
+import org.apache.omid.tso.ProgrammableTSOServer.Response.ResponseType;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.net.InetSocketAddress;
+import java.nio.channels.ClosedChannelException;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.Executors;
+
+/**
+ * Used in tests. Allows to program the set of responses returned by a TSO
+ */
+public class ProgrammableTSOServer extends SimpleChannelHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ProgrammableTSOServer.class);
+
+ private ChannelFactory factory;
+ private ChannelGroup channelGroup;
+
+ private Queue<Response> responseQueue = new LinkedList<>();
+
+ @Inject
+ public ProgrammableTSOServer(int port) {
+ // Setup netty listener
+ factory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(new ThreadFactoryBuilder()
+ .setNameFormat("boss-%d").build()), Executors.newCachedThreadPool(new ThreadFactoryBuilder()
+ .setNameFormat("worker-%d").build()), (Runtime.getRuntime().availableProcessors() * 2 + 1) * 2);
+
+ // Create the global ChannelGroup
+ channelGroup = new DefaultChannelGroup(ProgrammableTSOServer.class.getName());
+
+ ServerBootstrap bootstrap = new ServerBootstrap(factory);
+ bootstrap.setPipelineFactory(new TSOChannelHandler.TSOPipelineFactory(this));
+
+ // Add the parent channel to the group
+ Channel channel = bootstrap.bind(new InetSocketAddress(port));
+ channelGroup.add(channel);
+
+ LOG.info("********** Dumb TSO Server running on port {} **********", port);
+ }
+
+ // ************************* Main interface for tests *********************
+
+ /**
+ * Allows to add response to the queue of responses
+ *
+ * @param r
+ * the response to add
+ */
+ public void queueResponse(Response r) {
+ responseQueue.add(r);
+ }
+
+ /**
+ * Removes all the current responses in the queue
+ */
+ public void cleanResponses() {
+ responseQueue.clear();
+ }
+
+ // ******************** End of Main interface for tests *******************
+
+ @Override
+ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ channelGroup.add(ctx.getChannel());
+ }
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ channelGroup.remove(ctx.getChannel());
+ }
+
+ /**
+ * Handle received messages
+ */
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
+ Object msg = e.getMessage();
+ if (msg instanceof TSOProto.Request) {
+ TSOProto.Request request = (TSOProto.Request) msg;
+ Channel channel = ctx.getChannel();
+ if (request.hasHandshakeRequest()) {
+ checkHandshake(ctx, request.getHandshakeRequest());
+ return;
+ }
+ if (!handshakeCompleted(ctx)) {
+ LOG.info("handshake not completed");
+ channel.close();
+ }
+
+ Response resp = responseQueue.poll();
+ if (request.hasTimestampRequest()) {
+ if (resp == null || resp.type != ResponseType.TIMESTAMP) {
+ throw new IllegalStateException("Expecting TS response to send but got " + resp);
+ }
+ TimestampResponse tsResp = (TimestampResponse) resp;
+ sendTimestampResponse(tsResp.startTS, channel);
+ } else if (request.hasCommitRequest()) {
+ if (resp == null) {
+ throw new IllegalStateException("Expecting COMMIT response to send but got null");
+ }
+ switch (resp.type) {
+ case COMMIT:
+ CommitResponse commitResp = (CommitResponse) resp;
+ sendCommitResponse(commitResp.heuristicDecissionRequired,
+ commitResp.startTS,
+ commitResp.commitTS,
+ channel);
+ break;
+ case ABORT:
+ AbortResponse abortResp = (AbortResponse) resp;
+ sendAbortResponse(abortResp.startTS, channel);
+ break;
+ default:
+ throw new IllegalStateException("Expecting COMMIT response to send but got " + resp.type);
+ }
+ } else {
+ LOG.error("Invalid request {}", request);
+ ctx.getChannel().close();
+ }
+ } else {
+ LOG.error("Unknown message type", msg);
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
+ if (e.getCause() instanceof ClosedChannelException) {
+ return;
+ }
+ LOG.warn("TSOHandler: Unexpected exception from downstream.", e.getCause());
+ Channels.close(e.getChannel());
+ }
+
+ private void checkHandshake(final ChannelHandlerContext ctx, TSOProto.HandshakeRequest request) {
+ TSOProto.HandshakeResponse.Builder response = TSOProto.HandshakeResponse.newBuilder();
+ if (request.hasClientCapabilities()) {
+
+ response.setClientCompatible(true).setServerCapabilities(TSOProto.Capabilities.newBuilder().build());
+ TSOChannelContext tsoCtx = new TSOChannelContext();
+ tsoCtx.setHandshakeComplete();
+ ctx.setAttachment(tsoCtx);
+ } else {
+ response.setClientCompatible(false);
+ }
+ ctx.getChannel().write(TSOProto.Response.newBuilder().setHandshakeResponse(response.build()).build());
+ }
+
+ private boolean handshakeCompleted(ChannelHandlerContext ctx) {
+ Object o = ctx.getAttachment();
+ if (o instanceof TSOChannelContext) {
+ TSOChannelContext tsoCtx = (TSOChannelContext) o;
+ return tsoCtx.getHandshakeComplete();
+ }
+ return false;
+ }
+
+ private void sendTimestampResponse(long startTimestamp, Channel c) {
+ TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
+ TSOProto.TimestampResponse.Builder respBuilder = TSOProto.TimestampResponse.newBuilder();
+ respBuilder.setStartTimestamp(startTimestamp);
+ builder.setTimestampResponse(respBuilder.build());
+ c.write(builder.build());
+ }
+
+ private void sendCommitResponse(boolean makeHeuristicDecission, long startTimestamp, long commitTimestamp, Channel c) {
+ TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
+ TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
+ if (makeHeuristicDecission) { // If the commit is ambiguous is due to a new master TSO
+ commitBuilder.setMakeHeuristicDecision(true);
+ }
+ commitBuilder.setAborted(false).setStartTimestamp(startTimestamp).setCommitTimestamp(commitTimestamp);
+ builder.setCommitResponse(commitBuilder.build());
+ c.write(builder.build());
+ }
+
+ private void sendAbortResponse(long startTimestamp, Channel c) {
+ TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
+ TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
+ commitBuilder.setAborted(true).setStartTimestamp(startTimestamp);
+ builder.setCommitResponse(commitBuilder.build());
+ c.write(builder.build());
+ }
+
+ private static class TSOChannelContext {
+ boolean handshakeComplete;
+
+ TSOChannelContext() {
+ handshakeComplete = false;
+ }
+
+ boolean getHandshakeComplete() {
+ return handshakeComplete;
+ }
+
+ void setHandshakeComplete() {
+ handshakeComplete = true;
+ }
+ }
+
+ public static class TimestampResponse extends Response {
+
+ final long startTS;
+
+ public TimestampResponse(long startTS) {
+ super(ResponseType.TIMESTAMP);
+ this.startTS = startTS;
+ }
+
+ }
+
+ public static class CommitResponse extends Response {
+
+ final boolean heuristicDecissionRequired;
+ final long startTS;
+ final long commitTS;
+
+ public CommitResponse(boolean heuristicDecissionRequired, long startTS, long commitTS) {
+ super(ResponseType.COMMIT);
+ this.heuristicDecissionRequired = heuristicDecissionRequired;
+ this.startTS = startTS;
+ this.commitTS = commitTS;
+ }
+
+ }
+
+ public static class AbortResponse extends Response {
+
+ final long startTS;
+
+ public AbortResponse(long startTS) {
+ super(ResponseType.ABORT);
+ this.startTS = startTS;
+ }
+
+ }
+
+ abstract static class Response {
+
+ enum ResponseType {
+ TIMESTAMP, COMMIT, ABORT;
+ }
+
+ final ResponseType type;
+
+ public Response(ResponseType type) {
+ this.type = type;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TSOMockModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TSOMockModule.java b/tso-server/src/test/java/org/apache/omid/tso/TSOMockModule.java
new file mode 100644
index 0000000..9f3629a
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TSOMockModule.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.InMemoryCommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.apache.omid.tso.TimestampOracleImpl.InMemoryTimestampStorage;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+public class TSOMockModule extends AbstractModule {
+
+ private final TSOServerConfig config;
+
+ public TSOMockModule(TSOServerConfig config) {
+ this.config = config;
+ }
+
+ @Override
+ protected void configure() {
+
+ bind(TSOChannelHandler.class).in(Singleton.class);
+ bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
+ bind(CommitTable.class).to(InMemoryCommitTable.class).in(Singleton.class);
+ bind(TimestampStorage.class).to(InMemoryTimestampStorage.class).in(Singleton.class);
+ bind(TimestampOracle.class).to(PausableTimestampOracle.class).in(Singleton.class);
+ bind(Panicker.class).to(MockPanicker.class).in(Singleton.class);
+
+ install(config.getLeaseModule());
+ install(new DisruptorModule());
+
+ }
+
+ @Provides
+ TSOServerConfig provideTSOServerConfig() {
+ return config;
+ }
+
+ @Provides
+ @Singleton
+ MetricsRegistry provideMetricsRegistry() {
+ return new NullMetricsProvider();
+ }
+
+ @Provides
+ @Named(TSO_HOST_AND_PORT_KEY)
+ String provideTSOHostAndPort() throws SocketException, UnknownHostException {
+ return NetworkInterfaceUtils.getTSOHostAndPort(config);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TSOServerConfigTest.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TSOServerConfigTest.java b/tso-server/src/test/java/org/apache/omid/tso/TSOServerConfigTest.java
new file mode 100644
index 0000000..fd92718
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TSOServerConfigTest.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.testng.annotations.Test;
+
+public class TSOServerConfigTest {
+
+ @Test
+ public void testParsesOK() throws Exception {
+ new TSOServerConfig("test-omid.yml");
+ }
+
+}
\ No newline at end of file
[48/52] [abbrv] incubator-omid git commit: [ci skip]prepare release
omid-0.8.1.37
Posted by ik...@apache.org.
[ci skip]prepare release omid-0.8.1.37
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/052fb713
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/052fb713
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/052fb713
Branch: refs/heads/master
Commit: 052fb7134a50852e91ca6a658a8ae85fe8bb4eab
Parents: d2a26c4
Author: Omid CI <om...@yahoo-inc.com>
Authored: Tue Apr 19 21:28:28 2016 +0000
Committer: Omid CI <om...@yahoo-inc.com>
Committed: Tue Apr 19 21:28:28 2016 +0000
----------------------------------------------------------------------
benchmarks/pom.xml | 2 +-
codahale-metrics/pom.xml | 2 +-
commit-table/pom.xml | 2 +-
common/pom.xml | 2 +-
examples/pom.xml | 2 +-
hbase-client/pom.xml | 2 +-
hbase-commit-table/pom.xml | 2 +-
hbase-common/pom.xml | 2 +-
hbase-coprocessor/pom.xml | 2 +-
hbase-shims/hbase-0/pom.xml | 2 +-
hbase-shims/hbase-1/pom.xml | 2 +-
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 2 +-
metrics/pom.xml | 2 +-
pom.xml | 4 ++--
statemachine/pom.xml | 2 +-
timestamp-storage/pom.xml | 2 +-
transaction-client/pom.xml | 2 +-
tso-server/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index 1b8d49d..59cebbf 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>benchmarks</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index 1fddbcd..933749a 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>org.apache.omid</groupId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index 47afe60..a0adc3f 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 3952b3d..e20867e 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index a39d652..91dc26f 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>org.apache.omid</groupId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index a0869a5..74a6c66 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index c623ebf..e954ff0 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase-commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index d912e4c..89691d9 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index beec0b4..c632709 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase-coprocessor</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index b5c9044..950e79f 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase0-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index 11dd838..12c6e38 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase1-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index c8eddd1..a49e02f 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>omid-shims-aggregator</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index c28c507..cfa693d 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>hbase-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index e4dd5e0..776925c 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>org.apache.omid</groupId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>metrics</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 823db65..10c0062 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
<organization>
<name>Apache Software Foundation</name>
@@ -105,7 +105,7 @@
<connection>scm:git:git://github.com/yahoo/omid.git</connection>
<developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
<url>https://github.com/yahoo/omid</url>
- <tag>master</tag>
+ <tag>omid-0.8.1.37</tag>
</scm>
<distributionManagement>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index b240062..24fc52d 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<name>State Machine</name>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index 2120458..066d0e4 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>timestamp-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index e5a0d1e..fb8bd84 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>transaction-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/052fb713/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index f60e2f2..20077d8 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37-SNAPSHOT</version>
+ <version>0.8.1.37</version>
</parent>
<artifactId>tso-server</artifactId>
[35/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestEndToEndScenariosWithHA.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestEndToEndScenariosWithHA.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestEndToEndScenariosWithHA.java
new file mode 100644
index 0000000..2c7f9f9
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestEndToEndScenariosWithHA.java
@@ -0,0 +1,412 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Charsets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.omid.TestUtils;
+import org.apache.omid.tso.LeaseManagement;
+import org.apache.omid.tso.PausableLeaseManager;
+import org.apache.omid.tso.TSOServer;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.recipes.cache.NodeCache;
+import org.apache.curator.framework.recipes.cache.NodeCacheListener;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+import static org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
+import static org.apache.omid.tso.client.OmidClientConfiguration.ConnType.HA;
+import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+@Test(groups = "sharedHBase")
+public class TestEndToEndScenariosWithHA extends OmidTestBase {
+
+ private static final int TEST_LEASE_PERIOD_MS = 5_000;
+ private static final String CURRENT_TSO_PATH = "/CURRENT_TSO_PATH";
+ private static final String TSO_LEASE_PATH = "/TSO_LEASE_PATH";
+ private static final String NAMESPACE = "omid";
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestEndToEndScenariosWithHA.class);
+
+ private static final byte[] qualifier1 = Bytes.toBytes("test-q1");
+ private static final byte[] qualifier2 = Bytes.toBytes("test-q2l");
+ private static final byte[] row1 = Bytes.toBytes("row1");
+ private static final byte[] row2 = Bytes.toBytes("row2");
+ private static final byte[] initialData = Bytes.toBytes("testWrite-0");
+ private static final byte[] data1_q1 = Bytes.toBytes("testWrite-1-q1");
+ private static final byte[] data1_q2 = Bytes.toBytes("testWrite-1-q2");
+ private static final byte[] data2_q1 = Bytes.toBytes("testWrite-2-q1");
+ private static final byte[] data2_q2 = Bytes.toBytes("testWrite-2-q2");
+ private static final int TSO1_PORT = 2223;
+ private static final int TSO2_PORT = 4321;
+
+ private CountDownLatch barrierTillTSOAddressPublication;
+
+ private CuratorFramework zkClient;
+
+ private TSOServer tso1;
+ private TSOServer tso2;
+
+ private PausableLeaseManager leaseManager1;
+
+ private TransactionManager tm;
+
+ @BeforeMethod(alwaysRun = true, timeOut = 30_000)
+ public void setup() throws Exception {
+ // Get the zkConnection string from minicluster
+ String zkConnection = "localhost:" + hBaseUtils.getZkCluster().getClientPort();
+
+ zkClient = provideInitializedZookeeperClient(zkConnection);
+
+ // Synchronize TSO start
+ barrierTillTSOAddressPublication = new CountDownLatch(1);
+ final NodeCache currentTSOZNode = new NodeCache(zkClient, CURRENT_TSO_PATH);
+ currentTSOZNode.getListenable().addListener(new NodeCacheListener() {
+
+ @Override
+ public void nodeChanged() throws Exception {
+ byte[] currentTSOAndEpochAsBytes = currentTSOZNode.getCurrentData().getData();
+ String currentTSOAndEpoch = new String(currentTSOAndEpochAsBytes, Charsets.UTF_8);
+ if (currentTSOAndEpoch.endsWith("#0")) { // Wait till a TSO instance publishes the epoch
+ barrierTillTSOAddressPublication.countDown();
+ }
+ }
+
+ });
+ currentTSOZNode.start(true);
+
+ // Configure TSO 1
+ TSOServerConfig config1 = new TSOServerConfig();
+ config1.setPort(TSO1_PORT);
+ config1.setMaxItems(1000);
+ config1.setLeaseModule(new TestHALeaseManagementModule(TEST_LEASE_PERIOD_MS, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkConnection, NAMESPACE));
+ Injector injector1 = Guice.createInjector(new TestTSOModule(hbaseConf, config1));
+ LOG.info("===================== Starting TSO 1 =====================");
+ tso1 = injector1.getInstance(TSOServer.class);
+ leaseManager1 = (PausableLeaseManager) injector1.getInstance(LeaseManagement.class);
+ tso1.startAndWait();
+ TestUtils.waitForSocketListening("localhost", TSO1_PORT, 100);
+ LOG.info("================ Finished loading TSO 1 ==================");
+
+ // Configure TSO 2
+ TSOServerConfig config2 = new TSOServerConfig();
+ config2.setPort(TSO2_PORT);
+ config2.setMaxItems(1000);
+ config2.setLeaseModule(new TestHALeaseManagementModule(TEST_LEASE_PERIOD_MS, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkConnection, NAMESPACE));
+ Injector injector2 = Guice.createInjector(new TestTSOModule(hbaseConf, config2));
+ LOG.info("===================== Starting TSO 2 =====================");
+ tso2 = injector2.getInstance(TSOServer.class);
+ injector2.getInstance(LeaseManagement.class);
+ tso2.startAndWait();
+ // Don't do this here: TestUtils.waitForSocketListening("localhost", 4321, 100);
+ LOG.info("================ Finished loading TSO 2 ==================");
+
+ // Wait till the master TSO is up
+ barrierTillTSOAddressPublication.await();
+ currentTSOZNode.close();
+
+ // Configure HBase TM
+ LOG.info("===================== Starting TM =====================");
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionType(HA);
+ hbaseOmidClientConf.setConnectionString(zkConnection);
+ hbaseOmidClientConf.getOmidClientConfiguration().setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ hbaseOmidClientConf.getOmidClientConfiguration().setZkNamespace(NAMESPACE);
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ hbaseConf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 3);
+ tm = HBaseTransactionManager.builder(hbaseOmidClientConf).build();
+ LOG.info("===================== TM Started =========================");
+ }
+
+
+ @AfterMethod(alwaysRun = true, timeOut = 60_000)
+ public void cleanup() throws Exception {
+ LOG.info("Cleanup");
+ HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
+ deleteTable(admin, TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME));
+ hBaseUtils.createTable(Bytes.toBytes(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME),
+ new byte[][]{DEFAULT_TIMESTAMP_STORAGE_CF_NAME.getBytes()},
+ Integer.MAX_VALUE);
+ tso1.stopAndWait();
+ TestUtils.waitForSocketNotListening("localhost", TSO1_PORT, 100);
+ tso2.stopAndWait();
+ TestUtils.waitForSocketNotListening("localhost", TSO2_PORT, 100);
+
+ zkClient.delete().forPath(TSO_LEASE_PATH);
+ LOG.info("ZKPath {} deleted", TSO_LEASE_PATH);
+ zkClient.delete().forPath(CURRENT_TSO_PATH);
+ LOG.info("ZKPaths {} deleted", CURRENT_TSO_PATH);
+
+ zkClient.close();
+ }
+
+ //
+ // TSO 1 is MASTER & TSO 2 is BACKUP
+ // Setup: TX 0 -> Add initial data to cells R1C1 (v0) & R2C2 (v0)
+ // TX 1 starts (TSO1)
+ // TX 1 modifies cells R1C1 & R2C2 (v1)
+ // Interleaved Read TX -IR TX- starts (TSO1)
+ // TSO 1 PAUSES -> TSO 2 becomes MASTER
+ // IR TX reads R1C1 -> should get v0
+ // TX 1 tries to commit -> should abort because was started in TSO 1
+ // IR TX reads R2C2 -> should get v0
+ // IR TX tries to commit -> should abort because was started in TSO 1
+ // End of Test state: R1C1 & R2C2 (v0)
+ @Test(timeOut = 60_000)
+ public void testScenario1() throws Exception {
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Write initial values for the test
+ HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
+ long initialEpoch = tx0.getEpoch();
+ LOG.info("Starting Tx {} writing initial values for cells ({}) ", tx0, Bytes.toString(initialData));
+ Put putInitialDataRow1 = new Put(row1);
+ putInitialDataRow1.add(TEST_FAMILY.getBytes(), qualifier1, initialData);
+ txTable.put(tx0, putInitialDataRow1);
+ Put putInitialDataRow2 = new Put(row2);
+ putInitialDataRow2.add(TEST_FAMILY.getBytes(), qualifier2, initialData);
+ txTable.put(tx0, putInitialDataRow2);
+ tm.commit(tx0);
+
+ // Initial checks
+ checkRowValues(txTable, initialData, initialData);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx1, Bytes.toString(data1_q1),
+ Bytes.toString(data1_q2));
+ Put putData1R1Q1 = new Put(row1);
+ putData1R1Q1.add(TEST_FAMILY.getBytes(), qualifier1, data1_q1);
+ txTable.put(tx1, putData1R1Q1);
+ Put putData1R2Q2 = new Put(row2);
+ putData1R2Q2.add(TEST_FAMILY.getBytes(), qualifier2, data1_q2);
+ txTable.put(tx1, putData1R2Q2);
+
+ Transaction interleavedReadTx = tm.begin();
+
+ LOG.info("Starting Interleaving Read Tx {} for checking cell values", interleavedReadTx.getTransactionId());
+
+ // Simulate a GC pause to change mastership (should throw a ServiceUnavailable exception)
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ LOG.info("++++++++++++++++++++ PAUSING TSO 1 +++++++++++++++++++");
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ leaseManager1.pausedInStillInLeasePeriod();
+
+ // Read interleaved and check the values writen by tx 1
+ Get getRow1 = new Get(row1).setMaxVersions(1);
+ getRow1.addColumn(TEST_FAMILY.getBytes(), qualifier1);
+ Result r = txTable.get(interleavedReadTx, getRow1);
+ assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), initialData,
+ "Unexpected value for SI read R1Q1" + interleavedReadTx + ": "
+ + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));
+
+ // Try to commit, but it should abort due to the change in mastership
+ try {
+ tm.commit(tx1);
+ fail();
+ } catch (RollbackException e) {
+ // Expected
+ LOG.info("Rollback cause for Tx {}: ", tx1, e.getCause());
+ assertEquals(tx1.getStatus(), Transaction.Status.ROLLEDBACK);
+ assertEquals(tx1.getEpoch(), initialEpoch);
+ }
+
+ // Read interleaved and check the values written by tx 1
+ Get getRow2 = new Get(row2).setMaxVersions(1);
+ r = txTable.get(interleavedReadTx, getRow2);
+ assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), initialData,
+ "Unexpected value for SI read R2Q2" + interleavedReadTx + ": "
+ + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));
+
+ // Should commit because its a read only tx does not have to contact the TSO
+ tm.commit(interleavedReadTx);
+ assertEquals(interleavedReadTx.getEpoch(), initialEpoch);
+ assertEquals(interleavedReadTx.getStatus(), Transaction.Status.COMMITTED_RO);
+
+ LOG.info("Wait till the client is informed about the connection parameters of the new TSO");
+ TestUtils.waitForSocketListening("localhost", TSO2_PORT, 100);
+
+ checkRowValues(txTable, initialData, initialData);
+
+ // Need to resume to let other test progress
+ leaseManager1.resume();
+
+ }
+
+ }
+
+ //
+ // TSO 1 is MASTER & TSO 2 is BACKUP
+ // Setup: TX 0 -> Add initial data to cells R1C1 (v0) & R2C2 (v0)
+ // TX 1 starts (TSO1)
+ // TX 1 modifies cells R1C1 & R2C2 (v1)
+ // TSO 1 is KILLED -> TSO 2 becomes MASTER
+ // TX 1 tries to commit -> should abort because was started in TSO 1
+ // TX 2 starts (TSO1)
+ // TX 2 reads R1C1 -> should get v0
+ // TX 2 reads R2C2 -> should get v0
+ // TX 2 modifies cells R1C1 & R2C2 (v2)
+ // TX 2 commits
+ // End of Test state: R1C1 & R2C2 (v2)
+ @Test(timeOut = 60_000)
+ public void testScenario2() throws Exception {
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Write initial values for the test
+ HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
+ long initialEpoch = tx0.getEpoch();
+ LOG.info("Starting Tx {} writing initial values for cells ({}) ", tx0, Bytes.toString(initialData));
+ Put putInitialDataRow1 = new Put(row1);
+ putInitialDataRow1.add(TEST_FAMILY.getBytes(), qualifier1, initialData);
+ txTable.put(tx0, putInitialDataRow1);
+ Put putInitialDataRow2 = new Put(row2);
+ putInitialDataRow2.add(TEST_FAMILY.getBytes(), qualifier2, initialData);
+ txTable.put(tx0, putInitialDataRow2);
+ tm.commit(tx0);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx1, Bytes.toString(data1_q1),
+ Bytes.toString(data1_q2));
+ Put putData1R1Q1 = new Put(row1);
+ putData1R1Q1.add(TEST_FAMILY.getBytes(), qualifier1, data1_q1);
+ txTable.put(tx1, putData1R1Q1);
+ Put putData1R2Q2 = new Put(row2);
+ putData1R2Q2.add(TEST_FAMILY.getBytes(), qualifier2, data1_q2);
+ txTable.put(tx1, putData1R2Q2);
+
+ // Provoke change in mastership (should throw a Connection exception)
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ LOG.info("++++++++++++++++++++ KILLING TSO 1 +++++++++++++++++++");
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
+ tso1.stopAndWait();
+ TestUtils.waitForSocketNotListening("localhost", TSO1_PORT, 100);
+
+ // Try to commit, but it should abort due to the change in mastership
+ try {
+ tm.commit(tx1);
+ String failMsg = String.format("%s should not commit. Initial epoch was: %d", tx1, initialEpoch);
+ fail(failMsg);
+ } catch (RollbackException e) {
+ // Expected
+ LOG.info("Rollback cause for Tx {}: ", tx1, e.getCause());
+ assertEquals(tx1.getStatus(), Transaction.Status.ROLLEDBACK);
+ assertEquals(tx1.getEpoch(), initialEpoch);
+ }
+
+ LOG.info("Sleep some time till the client is informed about"
+ + "the new TSO connection parameters and how can connect");
+ TimeUnit.SECONDS.sleep(10 + 2);
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx2, Bytes.toString(data1_q1),
+ Bytes.toString(data1_q2));
+ Get getData1R1Q1 = new Get(row1).setMaxVersions(1);
+ Result r = txTable.get(tx2, getData1R1Q1);
+ assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), initialData,
+ "Unexpected value for SI read R1Q1" + tx2 + ": "
+ + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));
+ Get getData1R2Q2 = new Get(row2).setMaxVersions(1);
+ r = txTable.get(tx2, getData1R2Q2);
+ assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), initialData,
+ "Unexpected value for SI read R1Q1" + tx2 + ": "
+ + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));
+
+ Put putData2R1Q1 = new Put(row1);
+ putData2R1Q1.add(TEST_FAMILY.getBytes(), qualifier1, data2_q1);
+ txTable.put(tx2, putData2R1Q1);
+ Put putData2R2Q2 = new Put(row2);
+ putData2R2Q2.add(TEST_FAMILY.getBytes(), qualifier2, data2_q2);
+ txTable.put(tx2, putData2R2Q2);
+ // This one should commit in the new TSO
+ tm.commit(tx2);
+
+ assertEquals(tx2.getStatus(), Transaction.Status.COMMITTED);
+ assertTrue(tx2.getEpoch() > tx0.getCommitTimestamp());
+
+ checkRowValues(txTable, data2_q1, data2_q2);
+ }
+
+ }
+
+ private void checkRowValues(TTable txTable, byte[] expectedDataR1Q1, byte[] expectedDataR2Q2)
+ throws TransactionException, IOException, RollbackException {
+ Transaction readTx = tm.begin();
+ LOG.info("Starting Read Tx {} for checking cell values", readTx.getTransactionId());
+ Get getRow1 = new Get(row1).setMaxVersions(1);
+ getRow1.addColumn(TEST_FAMILY.getBytes(), qualifier1);
+ Result r = txTable.get(readTx, getRow1);
+ assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), expectedDataR1Q1,
+ "Unexpected value for SI read R1Q1" + readTx + ": " + Bytes
+ .toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));
+ Get getRow2 = new Get(row2).setMaxVersions(1);
+ r = txTable.get(readTx, getRow2);
+ assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), expectedDataR2Q2,
+ "Unexpected value for SI read R2Q2" + readTx + ": " + Bytes
+ .toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));
+ tm.commit(readTx);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helpers
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private static CuratorFramework provideInitializedZookeeperClient(String zkConnection) throws Exception {
+
+ LOG.info("Creating Zookeeper Client connecting to {}", zkConnection);
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ CuratorFramework zkClient = CuratorFrameworkFactory
+ .builder()
+ .namespace(NAMESPACE)
+ .connectString(zkConnection)
+ .retryPolicy(retryPolicy).build();
+
+ LOG.info("Connecting to ZK cluster {}", zkClient.getState());
+ zkClient.start();
+ zkClient.blockUntilConnected();
+ LOG.info("Connection to ZK cluster {}", zkClient.getState());
+
+ return zkClient;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestFilters.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestFilters.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestFilters.java
new file mode 100644
index 0000000..5e3d7c0
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestFilters.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.ValueFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNull;
+
+/**
+ * Tests to verify that Get and Scan filters still work with transactions tables
+ */
+@Test(groups = "sharedHBase")
+public class TestFilters extends OmidTestBase {
+
+ byte[] family = Bytes.toBytes(TEST_FAMILY);
+ private byte[] row1 = Bytes.toBytes("row1");
+ private byte[] row2 = Bytes.toBytes("row2");
+ private byte[] row3 = Bytes.toBytes("row3");
+ private byte[] prefix = Bytes.toBytes("foo");
+ private byte[] col1 = Bytes.toBytes("foobar");
+ private byte[] col2 = Bytes.toBytes("boofar");
+
+ @Test(timeOut = 60_000)
+ public void testGetWithColumnPrefixFilter(ITestContext context) throws Exception {
+ testGet(context, new ColumnPrefixFilter(prefix));
+ }
+
+ @Test(timeOut = 60_000)
+ public void testGetWithValueFilter(ITestContext context) throws Exception {
+ testGet(context, new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(col1)));
+ }
+
+ private void testGet(ITestContext context, Filter f) throws Exception {
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString("localhost:1234");
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+ PostCommitActions syncPostCommitter = spy(
+ new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ AbstractTransactionManager tm = HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .commitTableClient(commitTableClient)
+ .postCommitter(syncPostCommitter)
+ .build();
+
+ writeRows(table, tm, syncPostCommitter);
+
+ Transaction t = tm.begin();
+ Get g = new Get(row1);
+ g.setFilter(f);
+
+ Result r = table.get(t, g);
+ assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
+ assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
+
+ g = new Get(row2);
+ g.setFilter(f);
+ r = table.get(t, g);
+ assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
+ assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
+
+ g = new Get(row3);
+ g.setFilter(f);
+ r = table.get(t, g);
+ assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
+ }
+
+ @Test(timeOut = 60_000)
+ public void testScanWithColumnPrefixFilter(ITestContext context) throws Exception {
+ testScan(context, new ColumnPrefixFilter(prefix));
+ }
+
+ @Test(timeOut = 60_000)
+ public void testScanWithValueFilter(ITestContext context) throws Exception {
+ testScan(context, new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(col1)));
+ }
+
+ private void testScan(ITestContext context, Filter f) throws Exception {
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.getOmidClientConfiguration().setConnectionString("localhost:1234");
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+ PostCommitActions syncPostCommitter = spy(
+ new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ AbstractTransactionManager tm = HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .commitTableClient(commitTableClient)
+ .postCommitter(syncPostCommitter)
+ .build();
+
+ writeRows(table, tm, syncPostCommitter);
+
+ Transaction t = tm.begin();
+ Scan s = new Scan().setFilter(f);
+
+ ResultScanner rs = table.getScanner(t, s);
+
+ Result r = rs.next();
+ assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
+ assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
+
+ r = rs.next();
+ assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
+ assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
+
+ r = rs.next();
+ assertNull("Last row shouldn't exist", r);
+ }
+
+
+ private void writeRows(TTable table, TransactionManager tm, PostCommitActions postCommitter)
+ throws Exception {
+ // create normal row with both cells
+ Transaction t = tm.begin();
+ Put p = new Put(row1);
+ p.add(family, col1, col1);
+ p.add(family, col2, col2);
+ table.put(t, p);
+ tm.commit(t);
+
+ // create normal row, but fail to update shadow cells
+ doAnswer(new Answer<ListenableFuture<Void>>() {
+ public ListenableFuture<Void> answer(InvocationOnMock invocation) {
+ // Do not invoke the real method
+ return SettableFuture.create();
+ }
+ }).when(postCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ t = tm.begin();
+ p = new Put(row2);
+ p.add(family, col1, col1);
+ p.add(family, col2, col2);
+ table.put(t, p);
+ try {
+ tm.commit(t);
+ } catch (TransactionException e) {
+ // Expected, see comment above
+ }
+
+ // create normal row with only one cell
+ t = tm.begin();
+ p = new Put(row3);
+ p.add(family, col2, col2);
+ table.put(t, p);
+ try {
+ tm.commit(t);
+ } catch (TransactionException e) {
+ // Expected, see comment above
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestHALeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestHALeaseManagementModule.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestHALeaseManagementModule.java
new file mode 100644
index 0000000..761fa52
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestHALeaseManagementModule.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.timestamp.storage.ZKModule;
+import org.apache.omid.tso.LeaseManagement;
+import org.apache.omid.tso.Panicker;
+import org.apache.omid.tso.PausableLeaseManager;
+import org.apache.omid.tso.TSOChannelHandler;
+import org.apache.omid.tso.TSOStateManager;
+import org.apache.curator.framework.CuratorFramework;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+class TestHALeaseManagementModule extends AbstractModule {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestHALeaseManagementModule.class);
+ private final long leasePeriodInMs;
+ private final String tsoLeasePath;
+ private final String currentTsoPath;
+ private final String zkCluster;
+ private final String zkNamespace;
+
+ TestHALeaseManagementModule(long leasePeriodInMs, String tsoLeasePath, String currentTsoPath,
+ String zkCluster, String zkNamespace) {
+ this.leasePeriodInMs = leasePeriodInMs;
+ this.tsoLeasePath = tsoLeasePath;
+ this.currentTsoPath = currentTsoPath;
+ this.zkCluster = zkCluster;
+ this.zkNamespace = zkNamespace;
+ }
+
+ @Override
+ protected void configure() {
+ install(new ZKModule(zkCluster, zkNamespace));
+ }
+
+ @Provides
+ @Singleton
+ LeaseManagement provideLeaseManager(@Named(TSO_HOST_AND_PORT_KEY) String tsoHostAndPort,
+ TSOChannelHandler tsoChannelHandler,
+ TSOStateManager stateManager,
+ CuratorFramework zkClient,
+ Panicker panicker)
+ throws LeaseManagement.LeaseManagementException {
+
+ LOG.info("Connection to ZK cluster [{}]", zkClient.getState());
+ return new PausableLeaseManager(tsoHostAndPort, tsoChannelHandler, stateManager, leasePeriodInMs,
+ tsoLeasePath, currentTsoPath, zkClient, panicker);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseOmidClientConfiguration.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseOmidClientConfiguration.java
new file mode 100644
index 0000000..ff98c41
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseOmidClientConfiguration.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.junit.Assert;
+import org.testng.annotations.Test;
+
+public class TestHBaseOmidClientConfiguration {
+
+ @Test
+ public void testYamlReading() {
+ HBaseOmidClientConfiguration configuration = new HBaseOmidClientConfiguration();
+ Assert.assertNotNull(configuration.getCommitTableName());
+ Assert.assertNotNull(configuration.getHBaseConfiguration());
+ Assert.assertNotNull(configuration.getMetrics());
+ Assert.assertNotNull(configuration.getOmidClientConfiguration());
+ }
+
+ @Test
+ public void testYamlReadingFromFile() {
+ HBaseOmidClientConfiguration configuration = new HBaseOmidClientConfiguration("/test-hbase-omid-client-config.yml");
+ Assert.assertNotNull(configuration.getCommitTableName());
+ Assert.assertNotNull(configuration.getHBaseConfiguration());
+ Assert.assertNotNull(configuration.getMetrics());
+ Assert.assertNotNull(configuration.getOmidClientConfiguration());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionClient.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionClient.java
new file mode 100644
index 0000000..8be7950
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionClient.java
@@ -0,0 +1,465 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.transaction.HBaseTransactionManager.CommitTimestampLocatorImpl;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import java.util.Map;
+
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.CACHE;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.COMMIT_TABLE;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.NOT_PRESENT;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.SHADOW_CELL;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestHBaseTransactionClient extends OmidTestBase {
+
+ private static final byte[] row1 = Bytes.toBytes("test-is-committed1");
+ private static final byte[] row2 = Bytes.toBytes("test-is-committed2");
+ private static final byte[] family = Bytes.toBytes(TEST_FAMILY);
+ private static final byte[] qualifier = Bytes.toBytes("testdata");
+ private static final byte[] data1 = Bytes.toBytes("testWrite-1");
+
+ @Test(timeOut = 30_000)
+ public void testIsCommitted(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+ tm.commit(t1);
+
+ HBaseTransaction t2 = (HBaseTransaction) tm.begin();
+ put = new Put(row2);
+ put.add(family, qualifier, data1);
+ table.put(t2, put);
+ table.getHTable().flushCommits();
+
+ HBaseTransaction t3 = (HBaseTransaction) tm.begin();
+ put = new Put(row2);
+ put.add(family, qualifier, data1);
+ table.put(t3, put);
+ tm.commit(t3);
+
+ HTable htable = new HTable(hbaseConf, TEST_TABLE);
+ HBaseCellId hBaseCellId1 = new HBaseCellId(htable, row1, family, qualifier, t1.getStartTimestamp());
+ HBaseCellId hBaseCellId2 = new HBaseCellId(htable, row2, family, qualifier, t2.getStartTimestamp());
+ HBaseCellId hBaseCellId3 = new HBaseCellId(htable, row2, family, qualifier, t3.getStartTimestamp());
+
+ HBaseTransactionClient hbaseTm = (HBaseTransactionClient) newTransactionManager(context);
+ assertTrue("row1 should be committed", hbaseTm.isCommitted(hBaseCellId1));
+ assertFalse("row2 should not be committed for kv2", hbaseTm.isCommitted(hBaseCellId2));
+ assertTrue("row2 should be committed for kv3", hbaseTm.isCommitted(hBaseCellId3));
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCrashAfterCommit(ITestContext context) throws Exception {
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
+ AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+
+ // Test shadow cell are created properly
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+ try {
+ tm.commit(t1);
+ } catch (Exception e) { // (*) crash
+ // Do nothing
+ }
+
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(row1,
+ family,
+ qualifier,
+ t1.getStartTimestamp(),
+ new TTableCellGetterAdapter(table)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(row1,
+ family,
+ qualifier,
+ t1.getStartTimestamp(),
+ new TTableCellGetterAdapter(table)));
+
+ HTable htable = new HTable(hbaseConf, TEST_TABLE);
+ HBaseCellId hBaseCellId = new HBaseCellId(htable, row1, family, qualifier, t1.getStartTimestamp());
+
+ HBaseTransactionClient hbaseTm = (HBaseTransactionClient) newTransactionManager(context);
+ assertTrue("row1 should be committed", hbaseTm.isCommitted(hBaseCellId));
+ }
+
+ @Test(timeOut = 30_000)
+ public void testReadCommitTimestampFromCommitTable(ITestContext context) throws Exception {
+
+ final long NON_EXISTING_CELL_TS = 1000L;
+
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
+ AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ // Test that a non-existing cell timestamp returns an empty result
+ Optional<CommitTimestamp> optionalCT = tm.commitTableClient.getCommitTimestamp(NON_EXISTING_CELL_TS).get();
+ assertFalse(optionalCT.isPresent());
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+ // Test that we get an invalidation mark for an invalidated transaction
+
+ // Start a transaction and invalidate it before commiting it
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+
+ assertTrue(tm.commitTableClient.tryInvalidateTransaction(tx1.getStartTimestamp()).get());
+ optionalCT = tm.commitTableClient.getCommitTimestamp(tx1.getStartTimestamp()).get();
+ assertTrue(optionalCT.isPresent());
+ CommitTimestamp ct = optionalCT.get();
+ assertFalse(ct.isValid());
+ assertEquals(CommitTable.INVALID_TRANSACTION_MARKER, ct.getValue());
+ assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
+
+ // Finally test that we get the right commit timestamp for a committed tx
+ // that couldn't get
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Put otherPut = new Put(row1);
+ otherPut.add(family, qualifier, data1);
+ table.put(tx2, otherPut);
+ try {
+ tm.commit(tx2);
+ } catch (Exception e) { // (*) crash
+ // Do nothing
+ }
+
+ optionalCT = tm.commitTableClient.getCommitTimestamp(tx2.getStartTimestamp()).get();
+ assertTrue(optionalCT.isPresent());
+ ct = optionalCT.get();
+ assertTrue(ct.isValid());
+ assertEquals(tx2.getCommitTimestamp(), ct.getValue());
+ assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void testReadCommitTimestampFromShadowCell(ITestContext context) throws Exception {
+
+ final long NON_EXISTING_CELL_TS = 1L;
+
+ HBaseTransactionManager tm = (HBaseTransactionManager) newTransactionManager(context);
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Test first we can not found a non-existent cell ts
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier, NON_EXISTING_CELL_TS);
+ // Set an empty cache to allow to bypass the checking
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ Optional<CommitTimestamp> optionalCT = tm
+ .readCommitTimestampFromShadowCell(NON_EXISTING_CELL_TS, ctLocator);
+ assertFalse(optionalCT.isPresent());
+
+ // Then test that for a transaction committed, we get the right CT
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+ tm.commit(tx1);
+ // Upon commit, the commit data should be in the shadow cells, so test it
+ optionalCT = tm.readCommitTimestampFromShadowCell(tx1.getStartTimestamp(), ctLocator);
+ assertTrue(optionalCT.isPresent());
+ CommitTimestamp ct = optionalCT.get();
+ assertTrue(ct.isValid());
+ assertEquals(tx1.getCommitTimestamp(), ct.getValue());
+ assertTrue(ct.getLocation().compareTo(SHADOW_CELL) == 0);
+
+ }
+
+ }
+
+ // Tests step 1 in AbstractTransactionManager.locateCellCommitTimestamp()
+ @Test(timeOut = 30_000)
+ public void testCellCommitTimestampIsLocatedInCache(ITestContext context) throws Exception {
+
+ final long CELL_ST = 1L;
+ final long CELL_CT = 2L;
+
+ HBaseTransactionManager tm = (HBaseTransactionManager) newTransactionManager(context);
+
+ // Pre-load the element to look for in the cache
+ HTable table = new HTable(hbaseConf, TEST_TABLE);
+ HBaseCellId hBaseCellId = new HBaseCellId(table, row1, family, qualifier, CELL_ST);
+ Map<Long, Long> fakeCache = Maps.newHashMap();
+ fakeCache.put(CELL_ST, CELL_CT);
+
+ // Then test that locator finds it in the cache
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId, fakeCache);
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(CELL_ST, tm.tsoClient.getEpoch(), ctLocator);
+ assertTrue(ct.isValid());
+ assertEquals(ct.getValue(), CELL_CT);
+ assertTrue(ct.getLocation().compareTo(CACHE) == 0);
+
+ }
+
+ // Tests step 2 in AbstractTransactionManager.locateCellCommitTimestamp()
+ // Note: This test is very similar to testCrashAfterCommit() above so
+ // maybe we should merge them in this test, adding the missing assertions
+ @Test(timeOut = 30_000)
+ public void testCellCommitTimestampIsLocatedInCommitTable(ITestContext context) throws Exception {
+
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
+ AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+ // Commit a transaction that is broken on commit to avoid
+ // write to the shadow cells and avoid cleaning the commit table
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+ try {
+ tm.commit(tx1);
+ } catch (Exception e) { // (*) crash
+ // Do nothing
+ }
+
+ // Test the locator finds the appropriate data in the commit table
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
+ tx1.getStartTimestamp());
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
+ ctLocator);
+ assertTrue(ct.isValid());
+ long expectedCommitTS = tx1.getStartTimestamp() + 1;
+ assertEquals(expectedCommitTS, ct.getValue());
+ assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
+ }
+
+ }
+
+ // Tests step 3 in AbstractTransactionManager.locateCellCommitTimestamp()
+ @Test(timeOut = 30_000)
+ public void testCellCommitTimestampIsLocatedInShadowCells(ITestContext context) throws Exception {
+
+ HBaseTransactionManager tm = (HBaseTransactionManager) newTransactionManager(context);
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+ // Commit a transaction to add ST/CT in commit table
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+ tm.commit(tx1);
+ // Upon commit, the commit data should be in the shadow cells
+
+ // Test the locator finds the appropriate data in the shadow cells
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
+ tx1.getStartTimestamp());
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
+ ctLocator);
+ assertTrue(ct.isValid());
+ assertEquals(tx1.getCommitTimestamp(), ct.getValue());
+ assertTrue(ct.getLocation().compareTo(SHADOW_CELL) == 0);
+ }
+
+ }
+
+ // Tests step 4 in AbstractTransactionManager.locateCellCommitTimestamp()
+ @Test(timeOut = 30_000)
+ public void testCellFromTransactionInPreviousEpochGetsInvalidComitTimestamp(ITestContext context) throws Exception {
+
+ final long CURRENT_EPOCH_FAKE = 1000L;
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, commitTableClient));
+ // The following lines allow to reach step 4)
+ // in AbstractTransactionManager.locateCellCommitTimestamp()
+ SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
+ f.set(Optional.<CommitTimestamp>absent());
+ doReturn(f).when(commitTableClient).getCommitTimestamp(any(Long.class));
+ doReturn(Optional.<CommitTimestamp>absent()).when(tm).readCommitTimestampFromShadowCell(any(Long.class),
+ any(CommitTimestampLocator.class));
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Commit a transaction to add ST/CT in commit table
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+ tm.commit(tx1);
+ // Upon commit, the commit data should be in the shadow cells
+
+ // Test a transaction in the previous epoch gets an InvalidCommitTimestamp class
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
+ tx1.getStartTimestamp());
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ // Fake the current epoch to simulate a newer TSO
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), CURRENT_EPOCH_FAKE, ctLocator);
+ assertFalse(ct.isValid());
+ assertEquals(CommitTable.INVALID_TRANSACTION_MARKER, ct.getValue());
+ assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
+ }
+ }
+
+ // Tests step 5 in AbstractTransactionManager.locateCellCommitTimestamp()
+ @Test(timeOut = 30_000)
+ public void testCellCommitTimestampIsLocatedInCommitTableAfterNotBeingInvalidated(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, syncPostCommitter));
+
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+ // The next two lines avoid steps 2) and 3) and go directly to step 5)
+ // in AbstractTransactionManager.locateCellCommitTimestamp()
+ SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
+ f.set(Optional.<CommitTimestamp>absent());
+ doReturn(f).doCallRealMethod().when(commitTableClient).getCommitTimestamp(any(Long.class));
+ doReturn(Optional.<CommitTimestamp>absent()).when(tm).readCommitTimestampFromShadowCell(any(Long.class),
+ any(CommitTimestampLocator.class));
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Commit a transaction that is broken on commit to avoid
+ // write to the shadow cells and avoid cleaning the commit table
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+ try {
+ tm.commit(tx1);
+ } catch (Exception e) { // (*) crash
+ // Do nothing
+ }
+
+ // Test the locator finds the appropriate data in the commit table
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
+ tx1.getStartTimestamp());
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
+ ctLocator);
+ assertTrue(ct.isValid());
+ assertEquals(tx1.getCommitTimestamp(), ct.getValue());
+ assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
+ }
+
+ }
+
+ // Tests step 6 in AbstractTransactionManager.locateCellCommitTimestamp()
+ @Test(timeOut = 30_000)
+ public void testCellCommitTimestampIsLocatedInShadowCellsAfterNotBeingInvalidated(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, commitTableClient));
+ // The next two lines avoid steps 2), 3) and 5) and go directly to step 6)
+ // in AbstractTransactionManager.locateCellCommitTimestamp()
+ SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
+ f.set(Optional.<CommitTimestamp>absent());
+ doReturn(f).when(commitTableClient).getCommitTimestamp(any(Long.class));
+ doReturn(Optional.<CommitTimestamp>absent()).doCallRealMethod()
+ .when(tm).readCommitTimestampFromShadowCell(any(Long.class), any(CommitTimestampLocator.class));
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Commit a transaction to add ST/CT in commit table
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(tx1, put);
+ tm.commit(tx1);
+ // Upon commit, the commit data should be in the shadow cells
+
+ // Test the locator finds the appropriate data in the shadow cells
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
+ tx1.getStartTimestamp());
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
+ ctLocator);
+ assertTrue(ct.isValid());
+ assertEquals(tx1.getCommitTimestamp(), ct.getValue());
+ assertTrue(ct.getLocation().compareTo(SHADOW_CELL) == 0);
+ }
+
+ }
+
+ // Tests last step in AbstractTransactionManager.locateCellCommitTimestamp()
+ @Test(timeOut = 30_000)
+ public void testCTLocatorReturnsAValidCTWhenNotPresent(ITestContext context) throws Exception {
+
+ final long CELL_TS = 1L;
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, commitTableClient));
+ // The following lines allow to reach the last return statement
+ SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
+ f.set(Optional.<CommitTimestamp>absent());
+ doReturn(f).when(commitTableClient).getCommitTimestamp(any(Long.class));
+ doReturn(Optional.<CommitTimestamp>absent()).when(tm).readCommitTimestampFromShadowCell(any(Long.class),
+ any(CommitTimestampLocator.class));
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+ HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier, CELL_TS);
+ CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
+ Maps.<Long, Long>newHashMap());
+ CommitTimestamp ct = tm.locateCellCommitTimestamp(CELL_TS, tm.tsoClient.getEpoch(), ctLocator);
+ assertTrue(ct.isValid());
+ assertEquals(-1L, ct.getValue());
+ assertTrue(ct.getLocation().compareTo(NOT_PRESENT) == 0);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionManager.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionManager.java
new file mode 100644
index 0000000..d65d27e
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestHBaseTransactionManager.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.omid.tso.client.TSOClient;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anySetOf;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+// TODO These tests should be adapted to a future test for AbstractTransactionManager as they should be DB independent
+@Test(groups = "sharedHBase")
+public class TestHBaseTransactionManager extends OmidTestBase {
+
+ private static final int FAKE_EPOCH_INCREMENT = 100;
+
+ private final byte[] row1 = Bytes.toBytes(TestHBaseTransactionManager.class.getCanonicalName());
+ private final byte[] testFamily = Bytes.toBytes(TEST_FAMILY);
+ private final byte[] qualifier = Bytes.toBytes("TEST_Q");
+ private final byte[] data1 = Bytes.toBytes("test_data1");
+
+
+ @Test(timeOut = 20_000)
+ public void testTxManagerGetsTimestampsInTheRightEpoch(ITestContext context) throws Exception {
+
+ TSOClient tsoClient = spy(getClient(context));
+
+ long fakeEpoch = tsoClient.getNewStartTimestamp().get() + FAKE_EPOCH_INCREMENT;
+
+ // Modify the epoch before testing the begin method
+ doReturn(fakeEpoch).when(tsoClient).getEpoch();
+
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, tsoClient));
+
+ // Create a transaction with the initial setup and check that the TX id matches the fake epoch created
+ Transaction tx1 = tm.begin();
+ assertEquals(tx1.getTransactionId(), fakeEpoch);
+ verify(tsoClient, timeout(100).times(FAKE_EPOCH_INCREMENT)).getEpoch();
+
+ }
+
+ @Test(timeOut = 20_000)
+ public void testReadOnlyTransactionsDoNotContactTSOServer(ITestContext context) throws Exception {
+
+ final int EXPECTED_INVOCATIONS_FOR_COMMIT = 1; // Test specific checks
+
+ TSOClient tsoClient = spy(getClient(context));
+ TransactionManager tm = newTransactionManager(context, tsoClient);
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Add initial data in a transactional context
+ Transaction tx1 = tm.begin();
+ Put put = new Put(row1);
+ put.add(testFamily, qualifier, data1);
+ txTable.put(tx1, put);
+ tm.commit(tx1);
+
+ verify(tsoClient, times(EXPECTED_INVOCATIONS_FOR_COMMIT)).commit(anyLong(), anySetOf(HBaseCellId.class));
+
+ // Create a read-only tx and verify that commit has not been invoked again in the TSOClient
+ AbstractTransaction readOnlyTx = (AbstractTransaction) tm.begin();
+ Get get = new Get(row1);
+ Result r = txTable.get(readOnlyTx, get);
+ assertTrue(Bytes.equals(r.getValue(testFamily, qualifier), data1), "Wrong value for RO-TX " + readOnlyTx);
+ assertTrue(readOnlyTx.getWriteSet().isEmpty());
+ tm.commit(readOnlyTx);
+
+ verify(tsoClient, times(EXPECTED_INVOCATIONS_FOR_COMMIT)).commit(anyLong(), anySetOf(HBaseCellId.class));
+ assertEquals(readOnlyTx.getStatus(), Transaction.Status.COMMITTED_RO);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestMultiplePut.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestMultiplePut.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestMultiplePut.java
new file mode 100644
index 0000000..04689ac
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestMultiplePut.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestMultiplePut extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestMultiplePut.class);
+
+ private static final byte[] testTable = Bytes.toBytes(TEST_TABLE);
+ private static final byte[] family = Bytes.toBytes(TEST_FAMILY);
+ private static final byte[] col1 = Bytes.toBytes("col1");
+ private static final byte[] col2 = Bytes.toBytes("col2");
+ private static final byte[] data = Bytes.toBytes("testData");
+
+ @Test(timeOut = 30_000)
+ public void testMultiPutInTwoDifferentColsOfSameRowAreInTheTableAfterCommit(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ Transaction tx = tm.begin();
+
+ byte[] rowToAdd = Bytes.toBytes(1000);
+
+ Put put1 = new Put(rowToAdd);
+ put1.add(family, col1, data);
+ txTable.put(tx, put1);
+
+ Put put2 = new Put(rowToAdd);
+ put2.add(family, col2, data);
+ txTable.put(tx, put2);
+
+ tm.commit(tx);
+
+ assertTrue(verifyValue(testTable, rowToAdd, family, col1, data), "Invalid value in table");
+ assertTrue(verifyValue(testTable, rowToAdd, family, col2, data), "Invalid value in table");
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testManyManyPutsInDifferentRowsAreInTheTableAfterCommit(ITestContext context) throws Exception {
+
+ final int NUM_ROWS_TO_ADD = 50;
+
+ TransactionManager tm = newTransactionManager(context);
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ Transaction tx = tm.begin();
+
+ for (int i = 0; i <= NUM_ROWS_TO_ADD; i++) {
+ byte[] rowToAdd = Bytes.toBytes(i);
+ byte[] dataForRowCol = Bytes.toBytes("testData" + i);
+ Put put = new Put(rowToAdd);
+ put.add(family, col1, dataForRowCol);
+ txTable.put(tx, put);
+ }
+
+ tm.commit(tx);
+
+ // Check some of the added values are there in the table
+ byte[] rowToCheck = Bytes.toBytes(0);
+ byte[] dataToCheck = Bytes.toBytes("testData" + 0);
+ assertTrue(verifyValue(testTable, rowToCheck, family, col1, dataToCheck), "Invalid value in table");
+ rowToCheck = Bytes.toBytes(NUM_ROWS_TO_ADD / 2);
+ dataToCheck = Bytes.toBytes("testData" + (NUM_ROWS_TO_ADD / 2));
+ assertTrue(verifyValue(testTable, rowToCheck, family, col1, dataToCheck), "Invalid value in table");
+ rowToCheck = Bytes.toBytes(NUM_ROWS_TO_ADD);
+ dataToCheck = Bytes.toBytes("testData" + NUM_ROWS_TO_ADD);
+ assertTrue(verifyValue(testTable, rowToCheck, family, col1, dataToCheck), "Invalid value in table");
+
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void testGetFromNonExistentRowAfterMultiplePutsReturnsNoResult(ITestContext context) throws Exception {
+
+ final int NUM_ROWS_TO_ADD = 10;
+
+ TransactionManager tm = newTransactionManager(context);
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ Transaction tx = tm.begin();
+
+ for (int i = 0; i < NUM_ROWS_TO_ADD; i++) {
+ byte[] rowToAdd = Bytes.toBytes(i);
+ Put put = new Put(rowToAdd);
+ put.add(family, col1, Bytes.toBytes("testData" + i));
+ txTable.put(tx, put);
+ }
+
+ byte[] nonExistentRow = Bytes.toBytes(NUM_ROWS_TO_ADD + 5);
+ Get get = new Get(nonExistentRow);
+ Result result = txTable.get(tx, get);
+
+ assertTrue(result.isEmpty(), "Found a row that should not exist");
+
+ tm.commit(tx);
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestReadPath.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestReadPath.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestReadPath.java
new file mode 100644
index 0000000..dd621ca
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestReadPath.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestReadPath extends OmidTestBase {
+
+ final byte[] family = Bytes.toBytes(TEST_FAMILY);
+ final byte[] row = Bytes.toBytes("row");
+ private final byte[] col = Bytes.toBytes("col1");
+ final byte[] data = Bytes.toBytes("data");
+ private final byte[] uncommitted = Bytes.toBytes("uncommitted");
+
+ @Test
+ public void testReadInterleaved(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ // Put some data on the DB
+ Transaction t1 = tm.begin();
+ Transaction t2 = tm.begin();
+
+ Put put = new Put(row);
+ put.add(family, col, data);
+ table.put(t1, put);
+ tm.commit(t1);
+
+ Get get = new Get(row);
+ Result result = table.get(t2, get);
+ assertFalse("Should be unable to read column", result.containsColumn(family, col));
+ }
+
+ @Test
+ public void testReadWithSeveralUncommitted(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ // Put some data on the DB
+ Transaction t = tm.begin();
+ Put put = new Put(row);
+ put.add(family, col, data);
+ table.put(t, put);
+ tm.commit(t);
+ List<Transaction> running = new ArrayList<>();
+
+ // Shade the data with uncommitted data
+ for (int i = 0; i < 10; ++i) {
+ t = tm.begin();
+ put = new Put(row);
+ put.add(family, col, uncommitted);
+ table.put(t, put);
+ running.add(t);
+ }
+
+ // Try to read from row, it should ignore the uncommitted data and return the original committed value
+ t = tm.begin();
+ Get get = new Get(row);
+ Result result = table.get(t, get);
+ Cell cell = result.getColumnLatestCell(family, col);
+ assertNotNull("KeyValue is null", cell);
+ byte[] value = CellUtil.cloneValue(cell);
+ assertTrue("Read data doesn't match", Arrays.equals(data, value));
+ tm.commit(t);
+
+ table.close();
+
+ for (Transaction r : running) {
+ tm.rollback(r);
+ }
+
+ }
+
+}
[09/52] [abbrv] incubator-omid git commit: Rename tsoclient package
to tso.client
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java
new file mode 100644
index 0000000..9a68c39
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientRaw.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.yahoo.omid.proto.TSOProto;
+import com.yahoo.omid.proto.TSOProto.Response;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Raw client for communicating with tso server directly with protobuf messages
+ */
+public class TSOClientRaw {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOClientRaw.class);
+
+ private final BlockingQueue<SettableFuture<Response>> responseQueue
+ = new ArrayBlockingQueue<SettableFuture<Response>>(5);
+ private final Channel channel;
+
+ public TSOClientRaw(String host, int port) throws InterruptedException, ExecutionException {
+ // Start client with Nb of active threads = 3 as maximum.
+ ChannelFactory factory = new NioClientSocketChannelFactory(
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), 3);
+ // Create the bootstrap
+ ClientBootstrap bootstrap = new ClientBootstrap(factory);
+
+ InetSocketAddress addr = new InetSocketAddress(host, port);
+
+ ChannelPipeline pipeline = bootstrap.getPipeline();
+ pipeline.addLast("lengthbaseddecoder",
+ new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
+ pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+ pipeline.addLast("protobufdecoder",
+ new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
+ pipeline.addLast("protobufencoder", new ProtobufEncoder());
+
+ Handler handler = new Handler();
+ pipeline.addLast("handler", handler);
+
+ bootstrap.setOption("tcpNoDelay", true);
+ bootstrap.setOption("keepAlive", true);
+ bootstrap.setOption("reuseAddress", true);
+ bootstrap.setOption("connectTimeoutMillis", 100);
+
+ ChannelFuture channelFuture = bootstrap.connect(addr).await();
+ channel = channelFuture.getChannel();
+ }
+
+ public void write(TSOProto.Request request) {
+ channel.write(request);
+ }
+
+ public Future<Response> getResponse() throws InterruptedException {
+ SettableFuture<Response> future = SettableFuture.<Response>create();
+ responseQueue.put(future);
+ return future;
+ }
+
+ public void close() throws InterruptedException {
+ responseQueue.put(SettableFuture.<Response>create());
+ channel.close();
+ }
+
+ private class Handler extends SimpleChannelHandler {
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
+ LOG.info("Message received", e);
+ if (e.getMessage() instanceof Response) {
+ Response resp = (Response) e.getMessage();
+ try {
+ SettableFuture<Response> future = responseQueue.take();
+ future.set(resp);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted in handler", ie);
+ }
+ } else {
+ LOG.warn("Received unknown message", e.getMessage());
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
+ LOG.info("Exception received", e.getCause());
+ try {
+ SettableFuture<Response> future = responseQueue.take();
+ future.setException(e.getCause());
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted handling exception", ie);
+ }
+ }
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e)
+ throws Exception {
+ LOG.info("Disconnected");
+ try {
+ SettableFuture<Response> future = responseQueue.take();
+ future.setException(new ConnectionException());
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted handling exception", ie);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
new file mode 100644
index 0000000..c89d05e
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.collect.Sets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.yahoo.omid.TestUtils;
+import com.yahoo.omid.committable.CommitTable;
+import com.yahoo.omid.tso.TSOMockModule;
+import com.yahoo.omid.tso.TSOServer;
+import com.yahoo.omid.tso.TSOServerConfig;
+import com.yahoo.omid.tso.util.DummyCellIdImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertNotNull;
+
+public class TestIntegrationOfTSOClientServerBasicFunctionality {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestIntegrationOfTSOClientServerBasicFunctionality.class);
+
+ private static final String TSO_SERVER_HOST = "localhost";
+ private int tsoServerPortForTest;
+
+ // Cells for tests
+ private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
+ private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
+
+ // Required infrastructure for TSO tsoClient-server integration testing
+ private TSOServer tsoServer;
+ private TSOClient tsoClient;
+ private TSOClient justAnotherTSOClient;
+ private CommitTable.Client commitTableClient;
+
+ @BeforeClass
+ public void setup() throws Exception {
+
+ tsoServerPortForTest = TestUtils.getFreeLocalPort();
+
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setMaxItems(1000);
+ tsoConfig.setPort(tsoServerPortForTest);
+ Module tsoServerMockModule = new TSOMockModule(tsoConfig);
+ Injector injector = Guice.createInjector(tsoServerMockModule);
+
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ commitTableClient = commitTable.getClient();
+
+ LOG.info("==================================================================================================");
+ LOG.info("======================================= Init TSO Server ==========================================");
+ LOG.info("==================================================================================================");
+
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_SERVER_HOST, tsoServerPortForTest, 100);
+
+ LOG.info("==================================================================================================");
+ LOG.info("===================================== TSO Server Initialized =====================================");
+ LOG.info("==================================================================================================");
+
+ LOG.info("==================================================================================================");
+ LOG.info("======================================= Setup TSO Clients ========================================");
+ LOG.info("==================================================================================================");
+
+ // Configure direct connection to the server
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + tsoServerPortForTest);
+
+ tsoClient = TSOClient.newInstance(tsoClientConf);
+ justAnotherTSOClient = TSOClient.newInstance(tsoClientConf);
+
+ LOG.info("==================================================================================================");
+ LOG.info("===================================== TSO Clients Initialized ====================================");
+ LOG.info("==================================================================================================");
+
+ Thread.currentThread().setName("Test Thread");
+
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+
+ tsoClient.close().get();
+
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, tsoServerPortForTest, 1000);
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTimestampsOrderingGrowMonotonically() throws Exception {
+ long referenceTimestamp;
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ referenceTimestamp = startTsTx1;
+
+ long startTsTx2 = tsoClient.getNewStartTimestamp().get();
+ assertEquals(startTsTx2, ++referenceTimestamp, "Should grow monotonically");
+ assertTrue(startTsTx2 > startTsTx1, "Two timestamps obtained consecutively should grow");
+
+ long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1)).get();
+ assertEquals(commitTsTx2, ++referenceTimestamp, "Should grow monotonically");
+
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c2)).get();
+ assertEquals(commitTsTx1, ++referenceTimestamp, "Should grow monotonically");
+
+ long startTsTx3 = tsoClient.getNewStartTimestamp().get();
+ assertEquals(startTsTx3, ++referenceTimestamp, "Should grow monotonically");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSimpleTransactionWithNoWriteSetCanCommit() throws Exception {
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.<CellId>newHashSet()).get();
+ assertTrue(commitTsTx1 > startTsTx1);
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTransactionWithMassiveWriteSetCanCommit() throws Exception {
+ long startTs = tsoClient.getNewStartTimestamp().get();
+
+ Set<CellId> cells = new HashSet<>();
+ for (int i = 0; i < 1_000_000; i++) {
+ cells.add(new DummyCellIdImpl(i));
+ }
+
+ long commitTs = tsoClient.commit(startTs, cells).get();
+ assertTrue(commitTs > startTs, "Commit TS should be higher than Start TS");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testMultipleSerialCommitsDoNotConflict() throws Exception {
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
+ assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be greater than Start TS");
+
+ long startTsTx2 = tsoClient.getNewStartTimestamp().get();
+ assertTrue(startTsTx2 > commitTsTx1, "TS should grow monotonically");
+
+ long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
+ assertTrue(commitTsTx2 > startTsTx2, "Commit TS must be greater than Start TS");
+
+ long startTsTx3 = tsoClient.getNewStartTimestamp().get();
+ long commitTsTx3 = tsoClient.commit(startTsTx3, Sets.newHashSet(c2)).get();
+ assertTrue(commitTsTx3 > startTsTx3, "Commit TS must be greater than Start TS");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitWritesToCommitTable() throws Exception {
+ long startTsForTx1 = tsoClient.getNewStartTimestamp().get();
+ long startTsForTx2 = tsoClient.getNewStartTimestamp().get();
+ assertTrue(startTsForTx2 > startTsForTx1, "Start TS should grow");
+
+ assertFalse(commitTableClient.getCommitTimestamp(startTsForTx1).get().isPresent(),
+ "Commit TS for Tx1 shouldn't appear in Commit Table");
+
+ long commitTsForTx1 = tsoClient.commit(startTsForTx1, Sets.newHashSet(c1)).get();
+ assertTrue(commitTsForTx1 > startTsForTx1, "Commit TS should be higher than Start TS for the same tx");
+
+ Long commitTs1InCommitTable = commitTableClient.getCommitTimestamp(startTsForTx1).get().get().getValue();
+ assertNotNull("Tx is committed, should return as such from Commit Table", commitTs1InCommitTable);
+ assertEquals(commitTsForTx1, (long) commitTs1InCommitTable,
+ "getCommitTimestamp() & commit() should report same Commit TS value for same tx");
+ assertTrue(commitTs1InCommitTable > startTsForTx2, "Commit TS should be higher than tx's Start TS");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTwoConcurrentTxWithOverlappingWritesetsHaveConflicts() throws Exception {
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ long startTsTx2 = tsoClient.getNewStartTimestamp().get();
+ assertTrue(startTsTx2 > startTsTx1, "Second TX should have higher TS");
+
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
+ assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be higher than Start TS for the same tx");
+
+ try {
+ tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
+ Assert.fail("Second TX should fail on commit");
+ } catch (ExecutionException ee) {
+ assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void testConflictsAndMonotonicallyTimestampGrowthWithTwoDifferentTSOClients() throws Exception {
+ long startTsTx1Client1 = tsoClient.getNewStartTimestamp().get();
+ long startTsTx2Client1 = tsoClient.getNewStartTimestamp().get();
+ long startTsTx3Client1 = tsoClient.getNewStartTimestamp().get();
+
+ tsoClient.commit(startTsTx1Client1, Sets.newHashSet(c1)).get();
+ try {
+ tsoClient.commit(startTsTx3Client1, Sets.newHashSet(c1, c2)).get();
+ Assert.fail("Second commit should fail as conflicts with the previous concurrent one");
+ } catch (ExecutionException ee) {
+ assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
+ }
+ long startTsTx4Client2 = justAnotherTSOClient.getNewStartTimestamp().get();
+
+ assertFalse(commitTableClient.getCommitTimestamp(startTsTx3Client1).get().isPresent(), "Tx3 didn't commit");
+ long commitTSTx1 = commitTableClient.getCommitTimestamp(startTsTx1Client1).get().get().getValue();
+ assertTrue(commitTSTx1 > startTsTx2Client1, "Tx1 committed after Tx2 started");
+ assertTrue(commitTSTx1 < startTsTx4Client2, "Tx1 committed before Tx4 started on the other TSO Client");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java
new file mode 100644
index 0000000..581e5ce
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientConnectionToTSO.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.yahoo.omid.TestUtils;
+import com.yahoo.omid.tso.HALeaseManagementModule;
+import com.yahoo.omid.tso.TSOMockModule;
+import com.yahoo.omid.tso.TSOServer;
+import com.yahoo.omid.tso.TSOServerConfig;
+import com.yahoo.omid.tso.VoidLeaseManagementModule;
+import com.yahoo.statemachine.StateMachine.FsmImpl;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.utils.CloseableUtils;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class TestTSOClientConnectionToTSO {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientConnectionToTSO.class);
+
+ // Constants and variables for component connectivity
+ private static final String TSO_HOST = "localhost";
+ private static final String CURRENT_TSO_PATH = "/current_tso_path";
+ private static final String TSO_LEASE_PATH = "/tso_lease_path";
+
+ private int tsoPortForTest;
+ private String zkClusterForTest;
+
+ private Injector injector = null;
+
+ private TestingServer zkServer;
+
+ private CuratorFramework zkClient;
+ private TSOServer tsoServer;
+
+ @BeforeMethod
+ public void beforeMethod() throws Exception {
+
+ tsoPortForTest = TestUtils.getFreeLocalPort();
+
+ int zkPortForTest = TestUtils.getFreeLocalPort();
+ zkClusterForTest = TSO_HOST + ":" + zkPortForTest;
+ LOG.info("Starting ZK Server in port {}", zkPortForTest);
+ zkServer = TestUtils.provideTestingZKServer(zkPortForTest);
+ LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
+
+ zkClient = TestUtils.provideConnectedZKClient(zkClusterForTest);
+
+ Stat stat;
+ try {
+ zkClient.delete().forPath(CURRENT_TSO_PATH);
+ stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
+ assertNull(stat, CURRENT_TSO_PATH + " should not exist");
+ } catch (NoNodeException e) {
+ LOG.info("{} ZNode did not exist", CURRENT_TSO_PATH);
+ }
+
+ }
+
+ @AfterMethod
+ public void afterMethod() {
+
+ zkClient.close();
+
+ CloseableUtils.closeQuietly(zkServer);
+ zkServer = null;
+ LOG.info("ZK Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testUnsuccessfulConnectionToTSO() throws Exception {
+
+ // When no HA node for TSOServer is found & no host:port config exists
+ // we should get an exception when getting the client
+ try {
+ TSOClient.newInstance(new OmidClientConfiguration());
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSuccessfulConnectionToTSOWithHostAndPort() throws Exception {
+
+ // Launch a TSO WITHOUT publishing the address in HA...
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setMaxItems(1000);
+ tsoConfig.setPort(tsoPortForTest);
+ tsoConfig.setLeaseModule(new VoidLeaseManagementModule());
+ injector = Guice.createInjector(new TSOMockModule(tsoConfig));
+ LOG.info("Starting TSO");
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading TSO");
+
+ // When no HA node for TSOServer is found we should get a connection
+ // to the TSO through the host:port configured...
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString("localhost:" + tsoPortForTest);
+ tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // ... so we should get responses from the methods
+ Long startTS = tsoClient.getNewStartTimestamp().get();
+ LOG.info("Start TS {} ", startTS);
+ assertEquals(startTS.longValue(), 1);
+
+ // Close the tsoClient connection and stop the TSO Server
+ tsoClient.close().get();
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("TSO Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSuccessfulConnectionToTSOThroughZK() throws Exception {
+
+ // Launch a TSO publishing the address in HA...
+ TSOServerConfig config = new TSOServerConfig();
+ config.setMaxItems(1000);
+ config.setPort(tsoPortForTest);
+ config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
+ injector = Guice.createInjector(new TSOMockModule(config));
+ LOG.info("Starting TSO");
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading TSO");
+
+ waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
+
+ // When a HA node for TSOServer is found we should get a connection
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionType(OmidClientConfiguration.ConnType.HA);
+ tsoClientConf.setConnectionString(zkClusterForTest);
+ tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // ... so we should get responses from the methods
+ Long startTS = tsoClient.getNewStartTimestamp().get();
+ LOG.info("Start TS {} ", startTS);
+ assertEquals(startTS.longValue(), 1);
+
+ // Close the tsoClient connection and stop the TSO Server
+ tsoClient.close().get();
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("TSO Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSuccessOfTSOClientReconnectionsToARestartedTSOWithZKPublishing() throws Exception {
+
+ // Start a TSO with HA...
+ TSOServerConfig config = new TSOServerConfig();
+ config.setMaxItems(1000);
+ config.setPort(tsoPortForTest);
+ config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
+ injector = Guice.createInjector(new TSOMockModule(config));
+ LOG.info("Starting Initial TSO");
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading TSO");
+
+ waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
+
+ // Then create the TSO Client under test...
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionType(OmidClientConfiguration.ConnType.HA);
+ tsoClientConf.setConnectionString(zkClusterForTest);
+ tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // ... and check that initially we get responses from the methods
+ Long startTS = tsoClient.getNewStartTimestamp().get();
+ LOG.info("Start TS {} ", startTS);
+ assertEquals(startTS.longValue(), 1);
+
+ // Then stop the server...
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("Initial TSO Server Stopped");
+
+ Thread.sleep(1500); // ...allow the client to receive disconnection event...
+ // ... and check that we get a conn exception when trying to access the client
+ try {
+ startTS = tsoClient.getNewStartTimestamp().get();
+ fail();
+ } catch (ExecutionException e) {
+ LOG.info("Exception expected");
+ // Internal accessor to fsm to do the required checkings
+ FsmImpl fsm = (FsmImpl) tsoClient.fsm;
+ assertEquals(e.getCause().getClass(), ConnectionException.class);
+ assertTrue(fsm.getState().getClass().equals(TSOClient.ConnectionFailedState.class)
+ ||
+ fsm.getState().getClass().equals(TSOClient.DisconnectedState.class));
+ }
+
+ // After that, simulate that a new TSO has been launched...
+ Injector newInjector = Guice.createInjector(new TSOMockModule(config));
+ LOG.info("Re-Starting again the TSO");
+ tsoServer = newInjector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading restarted TSO");
+
+ // Finally re-check that, eventually, we can get a new value from the new TSO...
+ boolean reconnectionActive = false;
+ while (!reconnectionActive) {
+ try {
+ startTS = tsoClient.getNewStartTimestamp().get();
+ reconnectionActive = true;
+ } catch (ExecutionException e) {
+ // Expected
+ }
+ }
+ assertNotNull(startTS);
+
+ // ...and stop the server
+ tsoServer.stopAndWait();
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("Restarted TSO Server Stopped");
+ }
+
+ private void waitTillTsoRegisters(CuratorFramework zkClient) throws Exception {
+ while (true) {
+ try {
+ Stat stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
+ if (stat == null) {
+ continue;
+ }
+ LOG.info("TSO registered in HA with path {}={}", CURRENT_TSO_PATH, stat.toString());
+ if (stat.toString().length() == 0) {
+ continue;
+ }
+ return;
+ } catch (Exception e) {
+ LOG.debug("TSO still has not registered yet, sleeping...", e);
+ Thread.sleep(500);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
new file mode 100644
index 0000000..0e15261
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
@@ -0,0 +1,423 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.collect.Sets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import com.yahoo.omid.TestUtils;
+import com.yahoo.omid.committable.CommitTable;
+import com.yahoo.omid.proto.TSOProto;
+import com.yahoo.omid.tso.PausableTimestampOracle;
+import com.yahoo.omid.tso.TSOMockModule;
+import com.yahoo.omid.tso.TSOServer;
+import com.yahoo.omid.tso.TSOServerConfig;
+import com.yahoo.omid.tso.TimestampOracle;
+import com.yahoo.omid.tso.util.DummyCellIdImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class TestTSOClientRequestAndResponseBehaviours {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientRequestAndResponseBehaviours.class);
+
+ private static final String TSO_SERVER_HOST = "localhost";
+ private static final int TSO_SERVER_PORT = 1234;
+
+ private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
+ private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
+
+ private final static Set<CellId> testWriteSet = Sets.newHashSet(c1, c2);
+
+ private OmidClientConfiguration tsoClientConf;
+
+ // Required infrastructure for TSOClient test
+ private TSOServer tsoServer;
+ private PausableTimestampOracle pausableTSOracle;
+ private CommitTable commitTable;
+
+ @BeforeClass
+ public void setup() throws Exception {
+
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setMaxItems(1000);
+ tsoConfig.setPort(TSO_SERVER_PORT);
+ Module tsoServerMockModule = new TSOMockModule(tsoConfig);
+ Injector injector = Guice.createInjector(tsoServerMockModule);
+
+ LOG.info("==================================================================================================");
+ LOG.info("======================================= Init TSO Server ==========================================");
+ LOG.info("==================================================================================================");
+
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 100);
+
+ LOG.info("==================================================================================================");
+ LOG.info("===================================== TSO Server Initialized =====================================");
+ LOG.info("==================================================================================================");
+
+ pausableTSOracle = (PausableTimestampOracle) injector.getInstance(TimestampOracle.class);
+ commitTable = injector.getInstance(CommitTable.class);
+
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 1000);
+
+ }
+
+ @BeforeMethod
+ public void beforeMethod() {
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+
+ this.tsoClientConf = tsoClientConf;
+
+ }
+
+ @AfterMethod
+ public void afterMethod() {
+
+ pausableTSOracle.resume();
+
+ }
+
+ /**
+ * Test to ensure TSOClient timeouts are cancelled.
+ * At some point a bug was detected because the TSOClient timeouts were not cancelled, and as timestamp requests
+ * had no way to be correlated to timestamp responses, random requests were just timed out after a certain time.
+ * We send a lot of timestamp requests, and wait for them to complete.
+ * Ensure that the next request doesn't get hit by the timeouts of the previous
+ * requests. (i.e. make sure we cancel timeouts)
+ */
+ @Test(timeOut = 30_000)
+ public void testTimeoutsAreCancelled() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ int requestTimeoutInMs = 500;
+ int requestMaxRetries = 5;
+ LOG.info("Request timeout {} ms; Max retries {}", requestTimeoutInMs, requestMaxRetries);
+ Future<Long> f = null;
+ for (int i = 0; i < (requestMaxRetries * 10); i++) {
+ f = client.getNewStartTimestamp();
+ }
+ if (f != null) {
+ f.get();
+ }
+ pausableTSOracle.pause();
+ long msToSleep = ((long) (requestTimeoutInMs * 0.75));
+ LOG.info("Sleeping for {} ms", msToSleep);
+ TimeUnit.MILLISECONDS.sleep(msToSleep);
+ f = client.getNewStartTimestamp();
+ msToSleep = ((long) (requestTimeoutInMs * 0.9));
+ LOG.info("Sleeping for {} ms", msToSleep);
+ TimeUnit.MILLISECONDS.sleep(msToSleep);
+ LOG.info("Resuming");
+ pausableTSOracle.resume();
+ f.get();
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitGetsServiceUnavailableExceptionWhenCommunicationFails() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestMaxRetries(0);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ List<Long> startTimestamps = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ startTimestamps.add(client.getNewStartTimestamp().get());
+ }
+
+ pausableTSOracle.pause();
+
+ List<Future<Long>> futures = new ArrayList<>();
+ for (long s : startTimestamps) {
+ futures.add(client.commit(s, Sets.<CellId>newHashSet()));
+ }
+ TSOClientAccessor.closeChannel(client);
+
+ for (Future<Long> f : futures) {
+ try {
+ f.get();
+ fail("Shouldn't be able to complete");
+ } catch (ExecutionException ee) {
+ assertTrue(ee.getCause() instanceof ServiceUnavailableException,
+ "Should be a service unavailable exception");
+ }
+ }
+ }
+
+ /**
+ * Test that if a client tries to make a request without handshaking, it will be disconnected.
+ */
+ @Test(timeOut = 30_000)
+ public void testHandshakeBetweenOldClientAndCurrentServer() throws Exception {
+
+ TSOClientRaw raw = new TSOClientRaw(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ TSOProto.Request request = TSOProto.Request.newBuilder()
+ .setTimestampRequest(TSOProto.TimestampRequest.newBuilder().build())
+ .build();
+ raw.write(request);
+ try {
+ raw.getResponse().get();
+ fail("Channel should be closed");
+ } catch (ExecutionException ee) {
+ assertEquals(ee.getCause().getClass(), ConnectionException.class, "Should be channel closed exception");
+ }
+ raw.close();
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Test duplicate commits
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * This tests the case where messages arrive at the TSO out of order. This can happen in the case
+ * the channel get dropped and the retry is done in a new channel. However, the TSO will respond with
+ * aborted to the original message because the retry was already committed and it would be prohibitively
+ * expensive to check all non-retry requests to see if they are already committed. For this reason
+ * a client must ensure that if it is sending a retry due to a socket error, the previous channel
+ * must be entirely closed so that it will not actually receive the abort response. TCP guarantees
+ * that this doesn't happen in non-socket error cases.
+ *
+ */
+ @Test(timeOut = 30_000)
+ public void testOutOfOrderMessages() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long ts1 = client.getNewStartTimestamp().get();
+
+ TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
+ TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
+ assertFalse(response1.getCommitResponse().getAborted(), "Retry Transaction should commit");
+ assertTrue(response2.getCommitResponse().getAborted(), "Transaction should abort");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testDuplicateCommitAborting() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ long ts2 = client.getNewStartTimestamp().get();
+ client.commit(ts2, testWriteSet).get();
+
+ TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
+ TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
+ assertTrue(response1.getCommitResponse().getAborted(), "Transaction should abort");
+ assertTrue(response2.getCommitResponse().getAborted(), "Retry commit should abort");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testDuplicateCommit() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long ts1 = client.getNewStartTimestamp().get();
+
+ TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
+ TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
+ assertEquals(response2.getCommitResponse().getCommitTimestamp(),
+ response1.getCommitResponse().getCommitTimestamp(),
+ "Commit timestamp should be the same");
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Test TSOClient retry behaviour
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Test(timeOut = 30_000)
+ public void testCommitCanSucceedWhenChannelDisconnected() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ pausableTSOracle.pause();
+ TSOFuture<Long> future = client.commit(ts1, testWriteSet);
+ TSOClientAccessor.closeChannel(client);
+ pausableTSOracle.resume();
+ future.get();
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitCanSucceedWithMultipleTimeouts() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestTimeoutInMs(100);
+ testTSOClientConf.setRequestMaxRetries(10000);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ pausableTSOracle.pause();
+ TSOFuture<Long> future = client.commit(ts1, testWriteSet);
+ TimeUnit.SECONDS.sleep(1);
+ pausableTSOracle.resume();
+ future.get();
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitFailWhenTSOIsDown() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestTimeoutInMs(100);
+ testTSOClientConf.setRequestMaxRetries(10);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ pausableTSOracle.pause();
+ TSOFuture<Long> future = client.commit(ts1, testWriteSet);
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ assertEquals(e.getCause().getClass(), ServiceUnavailableException.class,
+ "Should be a ServiceUnavailableExeption");
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTimestampRequestSucceedWithMultipleTimeouts() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestTimeoutInMs(100);
+ testTSOClientConf.setRequestMaxRetries(10000);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ pausableTSOracle.pause();
+ Future<Long> future = client.getNewStartTimestamp();
+ TimeUnit.SECONDS.sleep(1);
+ pausableTSOracle.resume();
+ future.get();
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // The next 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
+ // (They exercise the communication protocol) TODO Remove???
+ // ----------------------------------------------------------------------------------------------------------------
+ @Test
+ public void testCommitTimestampPresentInCommitTableReturnsCommit() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long tx1ST = client.getNewStartTimestamp().get();
+
+ clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ assertFalse(response.getCommitResponse().getAborted(), "Transaction should be committed");
+ assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
+ assertEquals(response.getCommitResponse().getCommitTimestamp(), tx1ST + 1);
+ }
+
+ @Test
+ public void testInvalidCommitTimestampPresentInCommitTableReturnsAbort() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long tx1ST = client.getNewStartTimestamp().get();
+ // Invalidate the transaction
+ commitTable.getClient().tryInvalidateTransaction(tx1ST);
+
+ clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ assertTrue(response.getCommitResponse().getAborted(), "Transaction should be aborted");
+ assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
+ assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
+ }
+
+ @Test
+ public void testCommitTimestampNotPresentInCommitTableReturnsAnAbort() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long tx1ST = client.getNewStartTimestamp().get();
+
+ clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+
+ // Simulate remove entry from the commit table before exercise retry
+ commitTable.getClient().completeTransaction(tx1ST);
+
+ TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ assertTrue(response.getCommitResponse().getAborted(), "Transaction should abort");
+ assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
+ assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
+ }
+ // ----------------------------------------------------------------------------------------------------------------
+ // The previous 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
+ // (They exercise the communication protocol) TODO Remove???
+ // ----------------------------------------------------------------------------------------------------------------
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private TSOProto.Request createRetryCommitRequest(long ts) {
+ return createCommitRequest(ts, true, testWriteSet);
+ }
+
+ private TSOProto.Request createCommitRequest(long ts, boolean retry, Set<CellId> writeSet) {
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
+ commitBuilder.setStartTimestamp(ts);
+ commitBuilder.setIsRetry(retry);
+ for (CellId cell : writeSet) {
+ commitBuilder.addCellId(cell.getCellId());
+ }
+ return builder.setCommitRequest(commitBuilder.build()).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java
new file mode 100644
index 0000000..3c2aa1e
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestTSOClientResponseHandling.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.yahoo.omid.tso.ProgrammableTSOServer;
+import com.yahoo.omid.tso.ProgrammableTSOServer.AbortResponse;
+import com.yahoo.omid.tso.ProgrammableTSOServer.CommitResponse;
+import com.yahoo.omid.tso.ProgrammableTSOServer.TimestampResponse;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.Assert.assertEquals;
+
+public class TestTSOClientResponseHandling {
+
+ private static final int TSO_PORT = 4321;
+ private static final long START_TS = 1L;
+ private static final long COMMIT_TS = 2L;
+
+ private ProgrammableTSOServer tsoServer = new ProgrammableTSOServer(TSO_PORT);
+ // Client under test
+ private TSOClient tsoClient;
+
+ @BeforeClass
+ public void configureAndCreateClient() throws IOException, InterruptedException {
+
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString("localhost:" + TSO_PORT);
+ tsoClient = TSOClient.newInstance(tsoClientConf);
+ }
+
+ @BeforeMethod
+ public void reset() {
+ tsoServer.cleanResponses();
+ }
+
+ @Test
+ public void testTimestampRequestReceivingASuccessfulResponse() throws Exception {
+ // test request timestamp response returns a timestamp
+
+ // Program the TSO to return an ad-hoc Timestamp response
+ tsoServer.queueResponse(new TimestampResponse(START_TS));
+
+ long startTS = tsoClient.getNewStartTimestamp().get();
+ assertEquals(startTS, START_TS);
+ }
+
+ @Test
+ public void testCommitRequestReceivingAnAbortResponse() throws Exception {
+ // test commit request which is aborted on the server side
+ // (e.g. due to conflicts with other transaction) throws an
+ // execution exception with an AbortException as a cause
+
+ // Program the TSO to return an Abort response
+ tsoServer.queueResponse(new AbortResponse(START_TS));
+
+ try {
+ tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
+ } catch (ExecutionException ee) {
+ assertEquals(ee.getCause().getClass(), AbortException.class);
+ }
+ }
+
+ @Test
+ public void testCommitRequestReceivingASuccessfulResponse() throws Exception {
+ // test commit request which is successfully committed on the server
+ // side returns a commit timestamp
+
+ // Program the TSO to return an Commit response (with no required heuristic actions)
+ tsoServer.queueResponse(new CommitResponse(false, START_TS, COMMIT_TS));
+
+ long commitTS = tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
+ assertEquals(commitTS, COMMIT_TS);
+ }
+
+ @Test
+ public void testCommitRequestReceivingAHeuristicResponse() throws Exception {
+ // test commit request which needs heuristic actions from the client
+ // throws an execution exception with a NewTSOException as a cause
+
+ // Program the TSO to return an Commit response requiring heuristic actions
+ tsoServer.queueResponse(new CommitResponse(true, START_TS, COMMIT_TS));
+ try {
+ tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
+ } catch (ExecutionException ee) {
+ assertEquals(ee.getCause().getClass(), NewTSOException.class);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java
new file mode 100644
index 0000000..70dec4e
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TestUnconnectedTSOClient.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.yahoo.omid.tso.util.DummyCellIdImpl;
+import com.yahoo.statemachine.StateMachine.FsmImpl;
+import org.slf4j.Logger;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static org.slf4j.LoggerFactory.getLogger;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+/**
+ * Test the behavior of requests on a TSOClient component that is not connected to a TSO server.
+ */
+public class TestUnconnectedTSOClient {
+
+ private static final Logger LOG = getLogger(TestUnconnectedTSOClient.class);
+
+ private static final int TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST = 2;
+
+ @Test(timeOut = 30_000) // 30 secs
+ public void testRequestsDoneOnAnUnconnectedTSOClientAlwaysReturn() throws Exception {
+
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString("localhost:12345");
+ tsoClientConf.setReconnectionDelayInSecs(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST);
+
+ // Component under test
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // Internal accessor to fsm
+ FsmImpl fsm = (FsmImpl) tsoClient.fsm;
+
+ assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
+
+ // Test requests to the 3 relevant methods in TSO client
+
+ try {
+ tsoClient.getNewStartTimestamp().get();
+ fail();
+ } catch (ExecutionException e) {
+ LOG.info("Exception expected");
+ assertEquals(e.getCause().getClass(), ConnectionException.class);
+ TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
+ assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
+ }
+
+ try {
+ tsoClient.commit(1, newHashSet(new DummyCellIdImpl(0xdeadbeefL))).get();
+ fail();
+ } catch (ExecutionException e) {
+ LOG.info("Exception expected");
+ assertEquals(e.getCause().getClass(), ConnectionException.class);
+ TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
+ assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
+ }
+
+ tsoClient.close().get();
+ LOG.info("No exception expected");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
deleted file mode 100644
index d98b353..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientAccessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.yahoo.omid.tsoclient.TSOClient.ConnectedState;
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-
-public class TSOClientAccessor {
-
- public static void closeChannel(TSOClient tsoClient) throws InterruptedException {
- FsmImpl fsm = (FsmImpl) tsoClient.fsm;
- ConnectedState connectedState = (ConnectedState) fsm.getState();
- connectedState.channel.close().await();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
deleted file mode 100644
index eb0a49c..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientOneShot.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.proto.TSOProto.Response;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ExecutionException;
-
-/**
- * Communication endpoint for TSO clients.
- */
-public class TSOClientOneShot {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOClientOneShot.class);
-
- private final String host;
- private final int port;
-
- public TSOClientOneShot(String host, int port) {
-
- this.host = host;
- this.port = port;
-
- }
-
- public TSOProto.Response makeRequest(TSOProto.Request request)
- throws InterruptedException, ExecutionException {
- TSOClientRaw raw = new TSOClientRaw(host, port);
-
- // do handshake
- TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
- handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
- raw.write(TSOProto.Request.newBuilder()
- .setHandshakeRequest(handshake.build()).build());
- Response response = raw.getResponse().get();
- assert (response.getHandshakeResponse().getClientCompatible());
-
- raw.write(request);
- response = raw.getResponse().get();
-
- raw.close();
- return response;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
deleted file mode 100644
index e6f7ebe..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TSOClientRaw.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.proto.TSOProto.Response;
-import org.jboss.netty.bootstrap.ClientBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
-import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
-import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-/**
- * Raw client for communicating with tso server directly with protobuf messages
- */
-public class TSOClientRaw {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOClientRaw.class);
-
- private final BlockingQueue<SettableFuture<Response>> responseQueue
- = new ArrayBlockingQueue<SettableFuture<Response>>(5);
- private final Channel channel;
-
- public TSOClientRaw(String host, int port) throws InterruptedException, ExecutionException {
- // Start client with Nb of active threads = 3 as maximum.
- ChannelFactory factory = new NioClientSocketChannelFactory(
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), 3);
- // Create the bootstrap
- ClientBootstrap bootstrap = new ClientBootstrap(factory);
-
- InetSocketAddress addr = new InetSocketAddress(host, port);
-
- ChannelPipeline pipeline = bootstrap.getPipeline();
- pipeline.addLast("lengthbaseddecoder",
- new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
- pipeline.addLast("protobufdecoder",
- new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
- pipeline.addLast("protobufencoder", new ProtobufEncoder());
-
- Handler handler = new Handler();
- pipeline.addLast("handler", handler);
-
- bootstrap.setOption("tcpNoDelay", true);
- bootstrap.setOption("keepAlive", true);
- bootstrap.setOption("reuseAddress", true);
- bootstrap.setOption("connectTimeoutMillis", 100);
-
- ChannelFuture channelFuture = bootstrap.connect(addr).await();
- channel = channelFuture.getChannel();
- }
-
- public void write(TSOProto.Request request) {
- channel.write(request);
- }
-
- public Future<Response> getResponse() throws InterruptedException {
- SettableFuture<Response> future = SettableFuture.<Response>create();
- responseQueue.put(future);
- return future;
- }
-
- public void close() throws InterruptedException {
- responseQueue.put(SettableFuture.<Response>create());
- channel.close();
- }
-
- private class Handler extends SimpleChannelHandler {
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
- LOG.info("Message received", e);
- if (e.getMessage() instanceof Response) {
- Response resp = (Response) e.getMessage();
- try {
- SettableFuture<Response> future = responseQueue.take();
- future.set(resp);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted in handler", ie);
- }
- } else {
- LOG.warn("Received unknown message", e.getMessage());
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
- LOG.info("Exception received", e.getCause());
- try {
- SettableFuture<Response> future = responseQueue.take();
- future.setException(e.getCause());
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted handling exception", ie);
- }
- }
-
- @Override
- public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e)
- throws Exception {
- LOG.info("Disconnected");
- try {
- SettableFuture<Response> future = responseQueue.take();
- future.setException(new ConnectionException());
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted handling exception", ie);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
deleted file mode 100644
index 5185d83..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestIntegrationOfTSOClientServerBasicFunctionality.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.collect.Sets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-import static org.testng.AssertJUnit.assertNotNull;
-
-public class TestIntegrationOfTSOClientServerBasicFunctionality {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestIntegrationOfTSOClientServerBasicFunctionality.class);
-
- private static final String TSO_SERVER_HOST = "localhost";
- private int tsoServerPortForTest;
-
- // Cells for tests
- private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
- private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
-
- // Required infrastructure for TSO tsoClient-server integration testing
- private TSOServer tsoServer;
- private TSOClient tsoClient;
- private TSOClient justAnotherTSOClient;
- private CommitTable.Client commitTableClient;
-
- @BeforeClass
- public void setup() throws Exception {
-
- tsoServerPortForTest = TestUtils.getFreeLocalPort();
-
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setMaxItems(1000);
- tsoConfig.setPort(tsoServerPortForTest);
- Module tsoServerMockModule = new TSOMockModule(tsoConfig);
- Injector injector = Guice.createInjector(tsoServerMockModule);
-
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- commitTableClient = commitTable.getClient();
-
- LOG.info("==================================================================================================");
- LOG.info("======================================= Init TSO Server ==========================================");
- LOG.info("==================================================================================================");
-
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_SERVER_HOST, tsoServerPortForTest, 100);
-
- LOG.info("==================================================================================================");
- LOG.info("===================================== TSO Server Initialized =====================================");
- LOG.info("==================================================================================================");
-
- LOG.info("==================================================================================================");
- LOG.info("======================================= Setup TSO Clients ========================================");
- LOG.info("==================================================================================================");
-
- // Configure direct connection to the server
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + tsoServerPortForTest);
-
- tsoClient = TSOClient.newInstance(tsoClientConf);
- justAnotherTSOClient = TSOClient.newInstance(tsoClientConf);
-
- LOG.info("==================================================================================================");
- LOG.info("===================================== TSO Clients Initialized ====================================");
- LOG.info("==================================================================================================");
-
- Thread.currentThread().setName("Test Thread");
-
- }
-
- @AfterClass
- public void tearDown() throws Exception {
-
- tsoClient.close().get();
-
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, tsoServerPortForTest, 1000);
-
- }
-
- @Test(timeOut = 30_000)
- public void testTimestampsOrderingGrowMonotonically() throws Exception {
- long referenceTimestamp;
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- referenceTimestamp = startTsTx1;
-
- long startTsTx2 = tsoClient.getNewStartTimestamp().get();
- assertEquals(startTsTx2, ++referenceTimestamp, "Should grow monotonically");
- assertTrue(startTsTx2 > startTsTx1, "Two timestamps obtained consecutively should grow");
-
- long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1)).get();
- assertEquals(commitTsTx2, ++referenceTimestamp, "Should grow monotonically");
-
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c2)).get();
- assertEquals(commitTsTx1, ++referenceTimestamp, "Should grow monotonically");
-
- long startTsTx3 = tsoClient.getNewStartTimestamp().get();
- assertEquals(startTsTx3, ++referenceTimestamp, "Should grow monotonically");
- }
-
- @Test(timeOut = 30_000)
- public void testSimpleTransactionWithNoWriteSetCanCommit() throws Exception {
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.<CellId>newHashSet()).get();
- assertTrue(commitTsTx1 > startTsTx1);
- }
-
- @Test(timeOut = 30_000)
- public void testTransactionWithMassiveWriteSetCanCommit() throws Exception {
- long startTs = tsoClient.getNewStartTimestamp().get();
-
- Set<CellId> cells = new HashSet<>();
- for (int i = 0; i < 1_000_000; i++) {
- cells.add(new DummyCellIdImpl(i));
- }
-
- long commitTs = tsoClient.commit(startTs, cells).get();
- assertTrue(commitTs > startTs, "Commit TS should be higher than Start TS");
- }
-
- @Test(timeOut = 30_000)
- public void testMultipleSerialCommitsDoNotConflict() throws Exception {
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
- assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be greater than Start TS");
-
- long startTsTx2 = tsoClient.getNewStartTimestamp().get();
- assertTrue(startTsTx2 > commitTsTx1, "TS should grow monotonically");
-
- long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
- assertTrue(commitTsTx2 > startTsTx2, "Commit TS must be greater than Start TS");
-
- long startTsTx3 = tsoClient.getNewStartTimestamp().get();
- long commitTsTx3 = tsoClient.commit(startTsTx3, Sets.newHashSet(c2)).get();
- assertTrue(commitTsTx3 > startTsTx3, "Commit TS must be greater than Start TS");
- }
-
- @Test(timeOut = 30_000)
- public void testCommitWritesToCommitTable() throws Exception {
- long startTsForTx1 = tsoClient.getNewStartTimestamp().get();
- long startTsForTx2 = tsoClient.getNewStartTimestamp().get();
- assertTrue(startTsForTx2 > startTsForTx1, "Start TS should grow");
-
- assertFalse(commitTableClient.getCommitTimestamp(startTsForTx1).get().isPresent(),
- "Commit TS for Tx1 shouldn't appear in Commit Table");
-
- long commitTsForTx1 = tsoClient.commit(startTsForTx1, Sets.newHashSet(c1)).get();
- assertTrue(commitTsForTx1 > startTsForTx1, "Commit TS should be higher than Start TS for the same tx");
-
- Long commitTs1InCommitTable = commitTableClient.getCommitTimestamp(startTsForTx1).get().get().getValue();
- assertNotNull("Tx is committed, should return as such from Commit Table", commitTs1InCommitTable);
- assertEquals(commitTsForTx1, (long) commitTs1InCommitTable,
- "getCommitTimestamp() & commit() should report same Commit TS value for same tx");
- assertTrue(commitTs1InCommitTable > startTsForTx2, "Commit TS should be higher than tx's Start TS");
- }
-
- @Test(timeOut = 30_000)
- public void testTwoConcurrentTxWithOverlappingWritesetsHaveConflicts() throws Exception {
- long startTsTx1 = tsoClient.getNewStartTimestamp().get();
- long startTsTx2 = tsoClient.getNewStartTimestamp().get();
- assertTrue(startTsTx2 > startTsTx1, "Second TX should have higher TS");
-
- long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
- assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be higher than Start TS for the same tx");
-
- try {
- tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
- Assert.fail("Second TX should fail on commit");
- } catch (ExecutionException ee) {
- assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
- }
- }
-
- @Test(timeOut = 30_000)
- public void testConflictsAndMonotonicallyTimestampGrowthWithTwoDifferentTSOClients() throws Exception {
- long startTsTx1Client1 = tsoClient.getNewStartTimestamp().get();
- long startTsTx2Client1 = tsoClient.getNewStartTimestamp().get();
- long startTsTx3Client1 = tsoClient.getNewStartTimestamp().get();
-
- tsoClient.commit(startTsTx1Client1, Sets.newHashSet(c1)).get();
- try {
- tsoClient.commit(startTsTx3Client1, Sets.newHashSet(c1, c2)).get();
- Assert.fail("Second commit should fail as conflicts with the previous concurrent one");
- } catch (ExecutionException ee) {
- assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
- }
- long startTsTx4Client2 = justAnotherTSOClient.getNewStartTimestamp().get();
-
- assertFalse(commitTableClient.getCommitTimestamp(startTsTx3Client1).get().isPresent(), "Tx3 didn't commit");
- long commitTSTx1 = commitTableClient.getCommitTimestamp(startTsTx1Client1).get().get().getValue();
- assertTrue(commitTSTx1 > startTsTx2Client1, "Tx1 committed after Tx2 started");
- assertTrue(commitTSTx1 < startTsTx4Client2, "Tx1 committed before Tx4 started on the other TSO Client");
- }
-
-}
[42/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
----------------------------------------------------------------------
diff --git a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java b/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
deleted file mode 100644
index 751ce34..0000000
--- a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-import com.codahale.metrics.ConsoleReporter;
-import com.codahale.metrics.CsvReporter;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Slf4jReporter;
-import com.codahale.metrics.Timer.Context;
-import com.codahale.metrics.graphite.Graphite;
-import com.codahale.metrics.graphite.GraphiteReporter;
-import com.google.common.base.Strings;
-import com.google.common.net.HostAndPort;
-import com.yahoo.omid.metrics.CodahaleMetricsConfig.Reporter;
-import org.apache.commons.io.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class CodahaleMetricsProvider implements MetricsProvider, MetricsRegistry {
-
- private static final Logger LOG = LoggerFactory.getLogger(CodahaleMetricsProvider.class);
-
- public static final Pattern CODAHALE_METRICS_CONFIG_PATTERN = Pattern
- .compile("(csv|slf4j|console|graphite):(.+):(\\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)");
-
- private MetricRegistry metrics = new MetricRegistry();
- private List<ScheduledReporter> reporters = new ArrayList<>();
-
- private final int metricsOutputFrequencyInSecs;
-
- public static CodahaleMetricsProvider createCodahaleMetricsProvider(
- List<String> metricsConfigs) throws IOException {
-
- CodahaleMetricsConfig codahaleConfig = new CodahaleMetricsConfig();
-
- for (String metricConfig : metricsConfigs) {
- Matcher matcher = CODAHALE_METRICS_CONFIG_PATTERN.matcher(metricConfig);
- if (matcher.matches()) {
-
- String reporter = matcher.group(1);
- String reporterConfig = matcher.group(2);
- codahaleConfig.setOutputFreqInSecs(Integer.valueOf(matcher.group(3)));
-
- switch (reporter) {
- case "csv":
- codahaleConfig.addReporter(Reporter.CSV);
- codahaleConfig.setCsvDir(reporterConfig);
- break;
- case "slf4j":
- codahaleConfig.addReporter(Reporter.SLF4J);
- codahaleConfig.setSlf4jLogger(reporterConfig);
- break;
- case "graphite":
- codahaleConfig.addReporter(Reporter.GRAPHITE);
- codahaleConfig.setGraphiteHostConfig(reporterConfig);
- break;
- case "console":
- codahaleConfig.addReporter(Reporter.CONSOLE);
- break;
- default:
- LOG.warn("Reporter {} unknown", reporter);
- break;
- }
- } else {
- LOG.error("Pattern {} not recognized", metricConfig);
- }
- }
- CodahaleMetricsProvider provider = new CodahaleMetricsProvider(codahaleConfig);
- provider.startMetrics();
- return provider;
- }
-
- public CodahaleMetricsProvider(CodahaleMetricsConfig conf) throws IOException {
- metricsOutputFrequencyInSecs = conf.getOutputFreqInSecs();
- int reporterCount = 0;
- for (Reporter reporter : conf.getReporters()) {
- ScheduledReporter codahaleReporter = null;
- switch (reporter) {
- case CONSOLE:
- codahaleReporter = createAndGetConfiguredConsoleReporter();
- break;
- case GRAPHITE:
- codahaleReporter = createAndGetConfiguredGraphiteReporter(conf.getPrefix(),
- conf.getGraphiteHostConfig());
- break;
- case CSV:
- codahaleReporter = createAndGetConfiguredCSVReporter(conf.getPrefix(),
- conf.getCsvDir());
- break;
- case SLF4J:
- codahaleReporter = createAndGetConfiguredSlf4jReporter(conf.getSlf4jLogger());
- break;
- }
- if (codahaleReporter != null) {
- reporters.add(codahaleReporter);
- reporterCount++;
- }
- }
- if (reporterCount == 0) {
- LOG.warn("No metric reporters found, so metrics won't be available");
- }
- startMetrics();
- }
-
- @Override
- public void startMetrics() {
- for (ScheduledReporter r : reporters) {
- LOG.info("Starting metrics reporter {} reporting every {} Secs",
- r.getClass().getCanonicalName(), metricsOutputFrequencyInSecs);
- r.start(metricsOutputFrequencyInSecs, TimeUnit.SECONDS);
- }
- }
-
- @Override
- public void stopMetrics() {
- for (ScheduledReporter r : reporters) {
- r.report();
- LOG.info("Stopping reporter {}", r.toString());
- r.stop();
- }
- }
-
- @Override
- public <T extends Number> void gauge(String name, Gauge<T> appGauge) {
- metrics.register(name, new CodahaleGauge<>(appGauge));
- }
-
- @Override
- public Counter counter(String name) {
- com.codahale.metrics.Counter counter = metrics.counter(name);
- return new CodahaleCounterWrapper(counter);
- }
-
-
- @Override
- public Timer timer(String name) {
- com.codahale.metrics.Timer timer = metrics.timer(name);
- return new CodahaleTimerWrapper(timer);
- }
-
- @Override
- public Meter meter(String name) {
- com.codahale.metrics.Meter meter = metrics.meter(name);
- return new CodahaleMeterWrapper(meter);
- }
-
- @Override
- public Histogram histogram(String name) {
- com.codahale.metrics.Histogram histogram = metrics.histogram(name);
- return new CodahaleHistogramWrapper(histogram);
- }
-
- private ScheduledReporter createAndGetConfiguredConsoleReporter() {
- return ConsoleReporter.forRegistry(metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- }
-
- private ScheduledReporter createAndGetConfiguredGraphiteReporter(String prefix, String graphiteHost) {
- LOG.info("Configuring Graphite reporter. Sendig data to host:port {}", graphiteHost);
- HostAndPort addr = HostAndPort.fromString(graphiteHost);
-
- final Graphite graphite = new Graphite(
- new InetSocketAddress(addr.getHostText(), addr.getPort()));
-
- return GraphiteReporter.forRegistry(metrics)
- .prefixedWith(prefix)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .filter(MetricFilter.ALL)
- .build(graphite);
- }
-
- private ScheduledReporter createAndGetConfiguredCSVReporter(String prefix, String csvDir) throws IOException {
- // NOTE:
- // 1) metrics output files are exclusive to a given process
- // 2) the output directory must exist
- // 3) if output files already exist they are not overwritten and there is no metrics output
- File outputDir;
- if (Strings.isNullOrEmpty(prefix)) {
- outputDir = new File(csvDir, prefix);
- } else {
- outputDir = new File(csvDir);
- }
- FileUtils.forceMkdir(outputDir);
- LOG.info("Configuring stats with csv output to directory [{}]", outputDir.getAbsolutePath());
- return CsvReporter.forRegistry(metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build(outputDir);
- }
-
-
- private ScheduledReporter createAndGetConfiguredSlf4jReporter(String slf4jLogger) {
- LOG.info("Configuring stats with SLF4J with logger {}", slf4jLogger);
- return Slf4jReporter.forRegistry(metrics)
- .outputTo(LoggerFactory.getLogger(slf4jLogger))
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- }
-
- /**
- * Metrics wrapper implementations
- */
-
- private class CodahaleGauge<T extends Number> implements com.codahale.metrics.Gauge<T> {
-
- private final Gauge<T> omidGauge;
-
- CodahaleGauge(Gauge<T> omidGauge) {
- this.omidGauge = omidGauge;
- }
-
- @Override
- public T getValue() {
- return omidGauge.getValue();
- }
-
- }
-
- private class CodahaleCounterWrapper implements Counter {
-
- private final com.codahale.metrics.Counter counter;
-
- CodahaleCounterWrapper(com.codahale.metrics.Counter counter) {
- this.counter = counter;
- }
-
- @Override
- public void inc() {
- counter.inc();
- }
-
- @Override
- public void inc(long n) {
- counter.inc(n);
- }
-
- @Override
- public void dec() {
- counter.dec();
- }
-
- @Override
- public void dec(long n) {
- counter.dec(n);
- }
-
- }
-
- private class CodahaleTimerWrapper implements Timer {
-
- private final com.codahale.metrics.Timer timer;
-
- private Context context;
-
- CodahaleTimerWrapper(com.codahale.metrics.Timer timer) {
- this.timer = timer;
- }
-
- @Override
- public void start() {
- context = timer.time();
- }
-
- @Override
- public void stop() {
- context.stop();
- }
-
- @Override
- public void update(long durationInNs) {
- timer.update(durationInNs, TimeUnit.NANOSECONDS);
- }
-
- }
-
- private class CodahaleMeterWrapper implements Meter {
-
- private com.codahale.metrics.Meter meter;
-
- CodahaleMeterWrapper(com.codahale.metrics.Meter meter) {
- this.meter = meter;
- }
-
- @Override
- public void mark() {
- meter.mark();
- }
-
- @Override
- public void mark(long n) {
- meter.mark(n);
- }
-
- }
-
- private class CodahaleHistogramWrapper implements Histogram {
-
- private com.codahale.metrics.Histogram histogram;
-
- CodahaleHistogramWrapper(com.codahale.metrics.Histogram histogram) {
- this.histogram = histogram;
- }
-
- @Override
- public void update(int value) {
- histogram.update(value);
- }
-
- @Override
- public void update(long value) {
- histogram.update(value);
- }
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsConfig.java
----------------------------------------------------------------------
diff --git a/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsConfig.java b/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsConfig.java
new file mode 100644
index 0000000..cc43de9
--- /dev/null
+++ b/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsConfig.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+import com.google.inject.Inject;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+import java.util.HashSet;
+import java.util.Set;
+
+@Singleton
+public class CodahaleMetricsConfig extends AbstractMetricsConfig {
+
+ public enum Reporter {
+ CSV, SLF4J, GRAPHITE, CONSOLE
+ }
+
+ private static final String DEFAULT_PREFIX = "omid";
+ private static final String DEFAULT_GRAPHITE_HOST = "localhost:2003";
+ private static final String DEFAULT_CSV_DIR = ".";
+ private static final String DEFAULT_SLF4J_LOGGER = "metrics";
+
+ private static final String METRICS_CODAHALE_PREFIX_KEY = "metrics.codahale.prefix";
+ private static final String METRICS_CODAHALE_REPORTERS_KEY = "metrics.codahale.reporters";
+ private static final String METRICS_CODAHALE_GRAPHITE_HOST_CONFIG = "metrics.codahale.graphite.host.config";
+ private static final String METRICS_CODAHALE_CSV_DIR = "metrics.codahale.cvs.dir";
+ private static final String METRICS_CODAHALE_SLF4J_LOGGER = "metrics.codahale.slf4j.logger";
+
+ private String prefix = DEFAULT_PREFIX;
+ private Set<Reporter> reporters = new HashSet<Reporter>();
+ private String graphiteHostConfig = DEFAULT_GRAPHITE_HOST;
+ private String csvDir = DEFAULT_CSV_DIR;
+ private String slf4jLogger = DEFAULT_SLF4J_LOGGER;
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ @Inject(optional = true)
+ public void setPrefix(@Named(METRICS_CODAHALE_PREFIX_KEY) String prefix) {
+ this.prefix = prefix;
+ }
+
+ public Set<Reporter> getReporters() {
+ return reporters;
+ }
+
+ @Inject(optional = true)
+ public void setReporters(@Named(METRICS_CODAHALE_REPORTERS_KEY) Set<Reporter> reporters) {
+ this.reporters = reporters;
+ }
+
+ public void addReporter(Reporter reporter) {
+ reporters.add(reporter);
+ }
+
+ public String getGraphiteHostConfig() {
+ return graphiteHostConfig;
+ }
+
+ @Inject(optional = true)
+ public void setGraphiteHostConfig(@Named(METRICS_CODAHALE_GRAPHITE_HOST_CONFIG) String graphiteHostConfig) {
+ this.graphiteHostConfig = graphiteHostConfig;
+ }
+
+ public String getCsvDir() {
+ return csvDir;
+ }
+
+ @Inject(optional = true)
+ public void setCsvDir(@Named(METRICS_CODAHALE_CSV_DIR) String csvDir) {
+ this.csvDir = csvDir;
+ }
+
+ public String getSlf4jLogger() {
+ return slf4jLogger;
+ }
+
+ @Inject(optional = true)
+ public void setSlf4jLogger(@Named(METRICS_CODAHALE_SLF4J_LOGGER) String slf4jLogger) {
+ this.slf4jLogger = slf4jLogger;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsProvider.java
----------------------------------------------------------------------
diff --git a/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsProvider.java b/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsProvider.java
new file mode 100644
index 0000000..89cc49b
--- /dev/null
+++ b/codahale-metrics/src/main/java/org/apache/omid/metrics/CodahaleMetricsProvider.java
@@ -0,0 +1,346 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.CsvReporter;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+import com.codahale.metrics.Timer.Context;
+import com.codahale.metrics.graphite.Graphite;
+import com.codahale.metrics.graphite.GraphiteReporter;
+import com.google.common.base.Strings;
+import com.google.common.net.HostAndPort;
+import org.apache.omid.metrics.CodahaleMetricsConfig.Reporter;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class CodahaleMetricsProvider implements MetricsProvider, MetricsRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CodahaleMetricsProvider.class);
+
+ public static final Pattern CODAHALE_METRICS_CONFIG_PATTERN = Pattern
+ .compile("(csv|slf4j|console|graphite):(.+):(\\d+):(DAYS|HOURS|MICROSECONDS|MILLISECONDS|MINUTES|NANOSECONDS|SECONDS)");
+
+ private MetricRegistry metrics = new MetricRegistry();
+ private List<ScheduledReporter> reporters = new ArrayList<>();
+
+ private final int metricsOutputFrequencyInSecs;
+
+ public static CodahaleMetricsProvider createCodahaleMetricsProvider(
+ List<String> metricsConfigs) throws IOException {
+
+ CodahaleMetricsConfig codahaleConfig = new CodahaleMetricsConfig();
+
+ for (String metricConfig : metricsConfigs) {
+ Matcher matcher = CODAHALE_METRICS_CONFIG_PATTERN.matcher(metricConfig);
+ if (matcher.matches()) {
+
+ String reporter = matcher.group(1);
+ String reporterConfig = matcher.group(2);
+ codahaleConfig.setOutputFreqInSecs(Integer.valueOf(matcher.group(3)));
+
+ switch (reporter) {
+ case "csv":
+ codahaleConfig.addReporter(Reporter.CSV);
+ codahaleConfig.setCsvDir(reporterConfig);
+ break;
+ case "slf4j":
+ codahaleConfig.addReporter(Reporter.SLF4J);
+ codahaleConfig.setSlf4jLogger(reporterConfig);
+ break;
+ case "graphite":
+ codahaleConfig.addReporter(Reporter.GRAPHITE);
+ codahaleConfig.setGraphiteHostConfig(reporterConfig);
+ break;
+ case "console":
+ codahaleConfig.addReporter(Reporter.CONSOLE);
+ break;
+ default:
+ LOG.warn("Reporter {} unknown", reporter);
+ break;
+ }
+ } else {
+ LOG.error("Pattern {} not recognized", metricConfig);
+ }
+ }
+ CodahaleMetricsProvider provider = new CodahaleMetricsProvider(codahaleConfig);
+ provider.startMetrics();
+ return provider;
+ }
+
+ public CodahaleMetricsProvider(CodahaleMetricsConfig conf) throws IOException {
+ metricsOutputFrequencyInSecs = conf.getOutputFreqInSecs();
+ int reporterCount = 0;
+ for (Reporter reporter : conf.getReporters()) {
+ ScheduledReporter codahaleReporter = null;
+ switch (reporter) {
+ case CONSOLE:
+ codahaleReporter = createAndGetConfiguredConsoleReporter();
+ break;
+ case GRAPHITE:
+ codahaleReporter = createAndGetConfiguredGraphiteReporter(conf.getPrefix(),
+ conf.getGraphiteHostConfig());
+ break;
+ case CSV:
+ codahaleReporter = createAndGetConfiguredCSVReporter(conf.getPrefix(),
+ conf.getCsvDir());
+ break;
+ case SLF4J:
+ codahaleReporter = createAndGetConfiguredSlf4jReporter(conf.getSlf4jLogger());
+ break;
+ }
+ if (codahaleReporter != null) {
+ reporters.add(codahaleReporter);
+ reporterCount++;
+ }
+ }
+ if (reporterCount == 0) {
+ LOG.warn("No metric reporters found, so metrics won't be available");
+ }
+ startMetrics();
+ }
+
+ @Override
+ public void startMetrics() {
+ for (ScheduledReporter r : reporters) {
+ LOG.info("Starting metrics reporter {} reporting every {} Secs",
+ r.getClass().getCanonicalName(), metricsOutputFrequencyInSecs);
+ r.start(metricsOutputFrequencyInSecs, TimeUnit.SECONDS);
+ }
+ }
+
+ @Override
+ public void stopMetrics() {
+ for (ScheduledReporter r : reporters) {
+ r.report();
+ LOG.info("Stopping reporter {}", r.toString());
+ r.stop();
+ }
+ }
+
+ @Override
+ public <T extends Number> void gauge(String name, Gauge<T> appGauge) {
+ metrics.register(name, new CodahaleGauge<>(appGauge));
+ }
+
+ @Override
+ public Counter counter(String name) {
+ com.codahale.metrics.Counter counter = metrics.counter(name);
+ return new CodahaleCounterWrapper(counter);
+ }
+
+
+ @Override
+ public Timer timer(String name) {
+ com.codahale.metrics.Timer timer = metrics.timer(name);
+ return new CodahaleTimerWrapper(timer);
+ }
+
+ @Override
+ public Meter meter(String name) {
+ com.codahale.metrics.Meter meter = metrics.meter(name);
+ return new CodahaleMeterWrapper(meter);
+ }
+
+ @Override
+ public Histogram histogram(String name) {
+ com.codahale.metrics.Histogram histogram = metrics.histogram(name);
+ return new CodahaleHistogramWrapper(histogram);
+ }
+
+ private ScheduledReporter createAndGetConfiguredConsoleReporter() {
+ return ConsoleReporter.forRegistry(metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+ private ScheduledReporter createAndGetConfiguredGraphiteReporter(String prefix, String graphiteHost) {
+ LOG.info("Configuring Graphite reporter. Sendig data to host:port {}", graphiteHost);
+ HostAndPort addr = HostAndPort.fromString(graphiteHost);
+
+ final Graphite graphite = new Graphite(
+ new InetSocketAddress(addr.getHostText(), addr.getPort()));
+
+ return GraphiteReporter.forRegistry(metrics)
+ .prefixedWith(prefix)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .filter(MetricFilter.ALL)
+ .build(graphite);
+ }
+
+ private ScheduledReporter createAndGetConfiguredCSVReporter(String prefix, String csvDir) throws IOException {
+ // NOTE:
+ // 1) metrics output files are exclusive to a given process
+ // 2) the output directory must exist
+ // 3) if output files already exist they are not overwritten and there is no metrics output
+ File outputDir;
+ if (Strings.isNullOrEmpty(prefix)) {
+ outputDir = new File(csvDir, prefix);
+ } else {
+ outputDir = new File(csvDir);
+ }
+ FileUtils.forceMkdir(outputDir);
+ LOG.info("Configuring stats with csv output to directory [{}]", outputDir.getAbsolutePath());
+ return CsvReporter.forRegistry(metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build(outputDir);
+ }
+
+
+ private ScheduledReporter createAndGetConfiguredSlf4jReporter(String slf4jLogger) {
+ LOG.info("Configuring stats with SLF4J with logger {}", slf4jLogger);
+ return Slf4jReporter.forRegistry(metrics)
+ .outputTo(LoggerFactory.getLogger(slf4jLogger))
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ }
+
+ /**
+ * Metrics wrapper implementations
+ */
+
+ private class CodahaleGauge<T extends Number> implements com.codahale.metrics.Gauge<T> {
+
+ private final Gauge<T> omidGauge;
+
+ CodahaleGauge(Gauge<T> omidGauge) {
+ this.omidGauge = omidGauge;
+ }
+
+ @Override
+ public T getValue() {
+ return omidGauge.getValue();
+ }
+
+ }
+
+ private class CodahaleCounterWrapper implements Counter {
+
+ private final com.codahale.metrics.Counter counter;
+
+ CodahaleCounterWrapper(com.codahale.metrics.Counter counter) {
+ this.counter = counter;
+ }
+
+ @Override
+ public void inc() {
+ counter.inc();
+ }
+
+ @Override
+ public void inc(long n) {
+ counter.inc(n);
+ }
+
+ @Override
+ public void dec() {
+ counter.dec();
+ }
+
+ @Override
+ public void dec(long n) {
+ counter.dec(n);
+ }
+
+ }
+
+ private class CodahaleTimerWrapper implements Timer {
+
+ private final com.codahale.metrics.Timer timer;
+
+ private Context context;
+
+ CodahaleTimerWrapper(com.codahale.metrics.Timer timer) {
+ this.timer = timer;
+ }
+
+ @Override
+ public void start() {
+ context = timer.time();
+ }
+
+ @Override
+ public void stop() {
+ context.stop();
+ }
+
+ @Override
+ public void update(long durationInNs) {
+ timer.update(durationInNs, TimeUnit.NANOSECONDS);
+ }
+
+ }
+
+ private class CodahaleMeterWrapper implements Meter {
+
+ private com.codahale.metrics.Meter meter;
+
+ CodahaleMeterWrapper(com.codahale.metrics.Meter meter) {
+ this.meter = meter;
+ }
+
+ @Override
+ public void mark() {
+ meter.mark();
+ }
+
+ @Override
+ public void mark(long n) {
+ meter.mark(n);
+ }
+
+ }
+
+ private class CodahaleHistogramWrapper implements Histogram {
+
+ private com.codahale.metrics.Histogram histogram;
+
+ CodahaleHistogramWrapper(com.codahale.metrics.Histogram histogram) {
+ this.histogram = histogram;
+ }
+
+ @Override
+ public void update(int value) {
+ histogram.update(value);
+ }
+
+ @Override
+ public void update(long value) {
+ histogram.update(value);
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index 4ad3c4d..ad5c100 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java b/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
deleted file mode 100644
index 78ae44b..0000000
--- a/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-public interface CommitTable {
-
- long INVALID_TRANSACTION_MARKER = -1L;
-
- Writer getWriter() throws IOException;
-
- Client getClient() throws IOException;
-
- interface Writer extends Closeable {
-
- void addCommittedTransaction(long startTimestamp, long commitTimestamp) throws IOException;
-
- void updateLowWatermark(long lowWatermark) throws IOException;
-
- /**
- * Flushes all the buffered events to the underlying datastore
- */
- void flush() throws IOException;
-
- /**
- * Allows to clean the write's current buffer. It is required for HA
- */
- void clearWriteBuffer();
- }
-
- interface Client extends Closeable {
-
- /**
- * Checks whether a transaction commit data is inside the commit table The function also checks whether the
- * transaction was invalidated and returns a commit timestamp type accordingly.
- *
- * @param startTimestamp the transaction start timestamp
- * @return Optional<CommitTimestamp> that represents a valid, invalid, or no timestamp.
- */
- ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp);
-
- ListenableFuture<Long> readLowWatermark();
-
- ListenableFuture<Void> completeTransaction(long startTimestamp);
-
- /**
- * Atomically tries to invalidate a non-committed transaction launched by a previous TSO server.
- *
- * @param startTimeStamp the transaction to invalidate
- * @return true on success and false on failure
- */
- ListenableFuture<Boolean> tryInvalidateTransaction(long startTimeStamp);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper classes
- // ----------------------------------------------------------------------------------------------------------------
- class CommitTimestamp {
-
- public enum Location {
- NOT_PRESENT, CACHE, COMMIT_TABLE, SHADOW_CELL
- }
-
- private final Location location;
- private final long value;
- private final boolean isValid;
-
- public CommitTimestamp(Location location, long value, boolean isValid) {
- this.location = location;
- this.value = value;
- this.isValid = isValid;
- }
-
- public Location getLocation() {
- return location;
- }
-
- public long getValue() {
- return value;
- }
-
- public boolean isValid() {
- return isValid;
- }
-
- @Override
- public String toString() {
- return String.format("Is valid=%s, Location=%s, Value=%d)", isValid, location, value);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java b/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
deleted file mode 100644
index 181e7aa..0000000
--- a/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location;
-
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class InMemoryCommitTable implements CommitTable {
-
- final ConcurrentHashMap<Long, Long> table = new ConcurrentHashMap<>();
-
- long lowWatermark;
-
- @Override
- public CommitTable.Writer getWriter() {
- return new Writer();
- }
-
- @Override
- public CommitTable.Client getClient() {
- return new Client();
- }
-
- public class Writer implements CommitTable.Writer {
- @Override
- public void addCommittedTransaction(long startTimestamp, long commitTimestamp) {
- // In this implementation, we use only one location that represents
- // both the value and the invalidation. Therefore, putIfAbsent is
- // required to make sure the entry was not invalidated.
- table.putIfAbsent(startTimestamp, commitTimestamp);
- }
-
- @Override
- public void updateLowWatermark(long lowWatermark) throws IOException {
- InMemoryCommitTable.this.lowWatermark = lowWatermark;
- }
-
- @Override
- public void flush() throws IOException {
- // noop
- }
-
- @Override
- public void clearWriteBuffer() {
- table.clear();
- }
-
- @Override
- public void close() {
- }
- }
-
- public class Client implements CommitTable.Client {
- @Override
- public ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp) {
- SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
- Long result = table.get(startTimestamp);
- if (result == null) {
- f.set(Optional.<CommitTimestamp>absent());
- } else {
- if (result == INVALID_TRANSACTION_MARKER) {
- f.set(Optional.of(new CommitTimestamp(Location.COMMIT_TABLE, INVALID_TRANSACTION_MARKER, false)));
- } else {
- f.set(Optional.of(new CommitTimestamp(Location.COMMIT_TABLE, result, true)));
- }
- }
- return f;
- }
-
- @Override
- public ListenableFuture<Long> readLowWatermark() {
- SettableFuture<Long> f = SettableFuture.create();
- f.set(lowWatermark);
- return f;
- }
-
- @Override
- public ListenableFuture<Void> completeTransaction(long startTimestamp) {
- SettableFuture<Void> f = SettableFuture.create();
- table.remove(startTimestamp);
- f.set(null);
- return f;
- }
-
- @Override
- public ListenableFuture<Boolean> tryInvalidateTransaction(long startTimestamp) {
-
- SettableFuture<Boolean> f = SettableFuture.create();
- Long old = table.get(startTimestamp);
-
- // If the transaction represented by startTimestamp is not in the map
- if (old == null) {
- // Try to invalidate the transaction
- old = table.putIfAbsent(startTimestamp, INVALID_TRANSACTION_MARKER);
- // If we were able to invalidate or someone else invalidate before us
- if (old == null || old == INVALID_TRANSACTION_MARKER) {
- f.set(true);
- return f;
- }
- } else {
- // Check if the value we read marked the transaction as invalid
- if (old == INVALID_TRANSACTION_MARKER) {
- f.set(true);
- return f;
- }
- }
-
- // At this point the transaction was already in the map at the beginning
- // of the method or was added right before we tried to invalidate.
- f.set(false);
- return f;
- }
-
- @Override
- public void close() {
- }
- }
-
- public int countElements() {
- return table.size();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java b/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
deleted file mode 100644
index 26aac54..0000000
--- a/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-
-import java.io.IOException;
-
-public class NullCommitTable implements CommitTable {
- @Override
- public CommitTable.Writer getWriter() {
- return new Writer();
- }
-
- @Override
- public CommitTable.Client getClient() {
- return new Client();
- }
-
- public class Writer implements CommitTable.Writer {
- @Override
- public void addCommittedTransaction(long startTimestamp, long commitTimestamp) {
- // noop
- }
-
- @Override
- public void updateLowWatermark(long lowWatermark) throws IOException {
- // noop
- }
-
- @Override
- public void clearWriteBuffer() {
- // noop
- }
-
- @Override
- public void flush() throws IOException {
- // noop
- }
-
- @Override
- public void close() {
- }
-
- }
-
- public static class Client implements CommitTable.Client {
- @Override
- public ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ListenableFuture<Long> readLowWatermark() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ListenableFuture<Void> completeTransaction(long startTimestamp) {
- SettableFuture<Void> f = SettableFuture.create();
- f.set(null);
- return f;
- }
-
- @Override
- public ListenableFuture<Boolean> tryInvalidateTransaction(long startTimestamp) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/main/java/org/apache/omid/committable/CommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/org/apache/omid/committable/CommitTable.java b/commit-table/src/main/java/org/apache/omid/committable/CommitTable.java
new file mode 100644
index 0000000..d47822b
--- /dev/null
+++ b/commit-table/src/main/java/org/apache/omid/committable/CommitTable.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+public interface CommitTable {
+
+ long INVALID_TRANSACTION_MARKER = -1L;
+
+ Writer getWriter() throws IOException;
+
+ Client getClient() throws IOException;
+
+ interface Writer extends Closeable {
+
+ void addCommittedTransaction(long startTimestamp, long commitTimestamp) throws IOException;
+
+ void updateLowWatermark(long lowWatermark) throws IOException;
+
+ /**
+ * Flushes all the buffered events to the underlying datastore
+ */
+ void flush() throws IOException;
+
+ /**
+ * Allows to clean the write's current buffer. It is required for HA
+ */
+ void clearWriteBuffer();
+ }
+
+ interface Client extends Closeable {
+
+ /**
+ * Checks whether a transaction commit data is inside the commit table The function also checks whether the
+ * transaction was invalidated and returns a commit timestamp type accordingly.
+ *
+ * @param startTimestamp the transaction start timestamp
+ * @return Optional<CommitTimestamp> that represents a valid, invalid, or no timestamp.
+ */
+ ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp);
+
+ ListenableFuture<Long> readLowWatermark();
+
+ ListenableFuture<Void> completeTransaction(long startTimestamp);
+
+ /**
+ * Atomically tries to invalidate a non-committed transaction launched by a previous TSO server.
+ *
+ * @param startTimeStamp the transaction to invalidate
+ * @return true on success and false on failure
+ */
+ ListenableFuture<Boolean> tryInvalidateTransaction(long startTimeStamp);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper classes
+ // ----------------------------------------------------------------------------------------------------------------
+ class CommitTimestamp {
+
+ public enum Location {
+ NOT_PRESENT, CACHE, COMMIT_TABLE, SHADOW_CELL
+ }
+
+ private final Location location;
+ private final long value;
+ private final boolean isValid;
+
+ public CommitTimestamp(Location location, long value, boolean isValid) {
+ this.location = location;
+ this.value = value;
+ this.isValid = isValid;
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+
+ public long getValue() {
+ return value;
+ }
+
+ public boolean isValid() {
+ return isValid;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Is valid=%s, Location=%s, Value=%d)", isValid, location, value);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/main/java/org/apache/omid/committable/InMemoryCommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/org/apache/omid/committable/InMemoryCommitTable.java b/commit-table/src/main/java/org/apache/omid/committable/InMemoryCommitTable.java
new file mode 100644
index 0000000..ceaec48
--- /dev/null
+++ b/commit-table/src/main/java/org/apache/omid/committable/InMemoryCommitTable.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.committable.CommitTable.CommitTimestamp.Location;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class InMemoryCommitTable implements CommitTable {
+
+ final ConcurrentHashMap<Long, Long> table = new ConcurrentHashMap<>();
+
+ long lowWatermark;
+
+ @Override
+ public CommitTable.Writer getWriter() {
+ return new Writer();
+ }
+
+ @Override
+ public CommitTable.Client getClient() {
+ return new Client();
+ }
+
+ public class Writer implements CommitTable.Writer {
+ @Override
+ public void addCommittedTransaction(long startTimestamp, long commitTimestamp) {
+ // In this implementation, we use only one location that represents
+ // both the value and the invalidation. Therefore, putIfAbsent is
+ // required to make sure the entry was not invalidated.
+ table.putIfAbsent(startTimestamp, commitTimestamp);
+ }
+
+ @Override
+ public void updateLowWatermark(long lowWatermark) throws IOException {
+ InMemoryCommitTable.this.lowWatermark = lowWatermark;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ // noop
+ }
+
+ @Override
+ public void clearWriteBuffer() {
+ table.clear();
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
+ public class Client implements CommitTable.Client {
+ @Override
+ public ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp) {
+ SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
+ Long result = table.get(startTimestamp);
+ if (result == null) {
+ f.set(Optional.<CommitTimestamp>absent());
+ } else {
+ if (result == INVALID_TRANSACTION_MARKER) {
+ f.set(Optional.of(new CommitTimestamp(Location.COMMIT_TABLE, INVALID_TRANSACTION_MARKER, false)));
+ } else {
+ f.set(Optional.of(new CommitTimestamp(Location.COMMIT_TABLE, result, true)));
+ }
+ }
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Long> readLowWatermark() {
+ SettableFuture<Long> f = SettableFuture.create();
+ f.set(lowWatermark);
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Void> completeTransaction(long startTimestamp) {
+ SettableFuture<Void> f = SettableFuture.create();
+ table.remove(startTimestamp);
+ f.set(null);
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Boolean> tryInvalidateTransaction(long startTimestamp) {
+
+ SettableFuture<Boolean> f = SettableFuture.create();
+ Long old = table.get(startTimestamp);
+
+ // If the transaction represented by startTimestamp is not in the map
+ if (old == null) {
+ // Try to invalidate the transaction
+ old = table.putIfAbsent(startTimestamp, INVALID_TRANSACTION_MARKER);
+ // If we were able to invalidate or someone else invalidate before us
+ if (old == null || old == INVALID_TRANSACTION_MARKER) {
+ f.set(true);
+ return f;
+ }
+ } else {
+ // Check if the value we read marked the transaction as invalid
+ if (old == INVALID_TRANSACTION_MARKER) {
+ f.set(true);
+ return f;
+ }
+ }
+
+ // At this point the transaction was already in the map at the beginning
+ // of the method or was added right before we tried to invalidate.
+ f.set(false);
+ return f;
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+
+ public int countElements() {
+ return table.size();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/main/java/org/apache/omid/committable/NullCommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/org/apache/omid/committable/NullCommitTable.java b/commit-table/src/main/java/org/apache/omid/committable/NullCommitTable.java
new file mode 100644
index 0000000..89e29eb
--- /dev/null
+++ b/commit-table/src/main/java/org/apache/omid/committable/NullCommitTable.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+
+import java.io.IOException;
+
+public class NullCommitTable implements CommitTable {
+ @Override
+ public CommitTable.Writer getWriter() {
+ return new Writer();
+ }
+
+ @Override
+ public CommitTable.Client getClient() {
+ return new Client();
+ }
+
+ public class Writer implements CommitTable.Writer {
+ @Override
+ public void addCommittedTransaction(long startTimestamp, long commitTimestamp) {
+ // noop
+ }
+
+ @Override
+ public void updateLowWatermark(long lowWatermark) throws IOException {
+ // noop
+ }
+
+ @Override
+ public void clearWriteBuffer() {
+ // noop
+ }
+
+ @Override
+ public void flush() throws IOException {
+ // noop
+ }
+
+ @Override
+ public void close() {
+ }
+
+ }
+
+ public static class Client implements CommitTable.Client {
+ @Override
+ public ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ListenableFuture<Long> readLowWatermark() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ListenableFuture<Void> completeTransaction(long startTimestamp) {
+ SettableFuture<Void> f = SettableFuture.create();
+ f.set(null);
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Boolean> tryInvalidateTransaction(long startTimestamp) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
----------------------------------------------------------------------
diff --git a/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java b/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
deleted file mode 100644
index 92cd248..0000000
--- a/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable;
-
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertNull;
-
-/**
- * TODO: Remove this class when removing this class from production code
- */
-public class NullCommitTableTest {
-
- private static final long TEST_ST = 1L;
- private static final long TEST_CT = 2L;
- private static final long TEST_LWM = 1L;
-
- @Test(timeOut = 10_000)
- public void testClientAndWriter() throws Exception {
-
- CommitTable commitTable = new NullCommitTable();
-
- try (CommitTable.Client commitTableClient = commitTable.getClient();
- CommitTable.Writer commitTableWriter = commitTable.getWriter()) {
-
- // Test client
- try {
- commitTableClient.readLowWatermark().get();
- } catch (UnsupportedOperationException e) {
- // expected
- }
-
- try {
- commitTableClient.getCommitTimestamp(TEST_ST).get();
- } catch (UnsupportedOperationException e) {
- // expected
- }
-
- try {
- commitTableClient.tryInvalidateTransaction(TEST_ST).get();
- } catch (UnsupportedOperationException e) {
- // expected
- }
-
- assertNull(commitTableClient.completeTransaction(TEST_ST).get());
-
- // Test writer
- commitTableWriter.updateLowWatermark(TEST_LWM);
- commitTableWriter.addCommittedTransaction(TEST_ST, TEST_CT);
- commitTableWriter.clearWriteBuffer();
- commitTableWriter.flush();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/commit-table/src/test/java/org/apache/omid/committable/NullCommitTableTest.java
----------------------------------------------------------------------
diff --git a/commit-table/src/test/java/org/apache/omid/committable/NullCommitTableTest.java b/commit-table/src/test/java/org/apache/omid/committable/NullCommitTableTest.java
new file mode 100644
index 0000000..612a4ed
--- /dev/null
+++ b/commit-table/src/test/java/org/apache/omid/committable/NullCommitTableTest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertNull;
+
+/**
+ * TODO: Remove this class when removing this class from production code
+ */
+public class NullCommitTableTest {
+
+ private static final long TEST_ST = 1L;
+ private static final long TEST_CT = 2L;
+ private static final long TEST_LWM = 1L;
+
+ @Test(timeOut = 10_000)
+ public void testClientAndWriter() throws Exception {
+
+ CommitTable commitTable = new NullCommitTable();
+
+ try (CommitTable.Client commitTableClient = commitTable.getClient();
+ CommitTable.Writer commitTableWriter = commitTable.getWriter()) {
+
+ // Test client
+ try {
+ commitTableClient.readLowWatermark().get();
+ } catch (UnsupportedOperationException e) {
+ // expected
+ }
+
+ try {
+ commitTableClient.getCommitTimestamp(TEST_ST).get();
+ } catch (UnsupportedOperationException e) {
+ // expected
+ }
+
+ try {
+ commitTableClient.tryInvalidateTransaction(TEST_ST).get();
+ } catch (UnsupportedOperationException e) {
+ // expected
+ }
+
+ assertNull(commitTableClient.completeTransaction(TEST_ST).get());
+
+ // Test writer
+ commitTableWriter.updateLowWatermark(TEST_LWM);
+ commitTableWriter.addCommittedTransaction(TEST_ST, TEST_CT);
+ commitTableWriter.clearWriteBuffer();
+ commitTableWriter.flush();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 5a84a35..d776e2b 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/main/java/com/yahoo/omid/YAMLUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/yahoo/omid/YAMLUtils.java b/common/src/main/java/com/yahoo/omid/YAMLUtils.java
deleted file mode 100644
index 6edd71d..0000000
--- a/common/src/main/java/com/yahoo/omid/YAMLUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid;
-
-import com.google.common.base.Preconditions;
-import com.google.common.io.Resources;
-import org.apache.commons.beanutils.BeanUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
-
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-
-public class YAMLUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(YAMLUtils.class);
-
- public void loadSettings(String resourcePath, String defaultResourcePath, Object bean) {
- try {
- Map properties = loadSettings(resourcePath, defaultResourcePath);
- BeanUtils.populate(bean, properties);
- } catch (IllegalAccessException | InvocationTargetException | IOException e) {
- throw new IllegalStateException(e);
- }
- }
-
- public void loadSettings(String resourcePath, Object bean) {
- try {
- Map properties = loadSettings(null, resourcePath);
- BeanUtils.populate(bean, properties);
- } catch (IllegalAccessException | InvocationTargetException | IOException e) {
- throw new IllegalStateException(e);
- }
- }
-
- @SuppressWarnings("unchecked")
- private Map loadSettings(String resourcePath, String defaultResourcePath) throws IOException {
- Map defaultSetting = loadAsMap(defaultResourcePath);
- Preconditions.checkState(defaultSetting.size() > 0, String.format("Failed to load file '%s' from classpath", defaultResourcePath));
- if (resourcePath != null) {
- Map userSetting = loadAsMap(resourcePath);
- defaultSetting.putAll(userSetting);
- }
- return defaultSetting;
- }
-
- @SuppressWarnings("unchecked")
- private Map loadAsMap(String path) throws IOException {
- try {
- String content = Resources.toString(Resources.getResource(path), Charset.forName("UTF-8"));
- LOG.debug("Loaded resource file '{}'\n{}", path, content);
- Map settings = new Yaml().loadAs(content, Map.class);
- if (settings == null) {
- settings = new HashMap(0);
- }
- return settings;
- } catch (IllegalArgumentException e) {
- return new HashMap();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java b/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
deleted file mode 100644
index 10e3df1..0000000
--- a/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.zk;
-
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-
-public class ZKUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(ZKUtils.class);
-
- public static CuratorFramework initZKClient(String zkCluster, String namespace, int zkConnectionTimeoutInSec)
- throws IOException, InterruptedException {
-
- LOG.info("Creating Zookeeper Client connecting to {}", zkCluster);
-
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- CuratorFramework zkClient = CuratorFrameworkFactory.builder()
- .namespace(namespace)
- .connectString(zkCluster)
- .retryPolicy(retryPolicy)
- .build();
-
- zkClient.start();
- if (zkClient.blockUntilConnected(zkConnectionTimeoutInSec, TimeUnit.SECONDS)) {
- LOG.info("Connected to ZK cluster '{}', client in state: [{}]", zkCluster, zkClient.getState());
- } else {
- throw new IOException(String.format("Can't contact ZK cluster '%s' after 10 seconds", zkCluster));
- }
-
- return zkClient;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/main/java/org/apache/omid/YAMLUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/omid/YAMLUtils.java b/common/src/main/java/org/apache/omid/YAMLUtils.java
new file mode 100644
index 0000000..156d033
--- /dev/null
+++ b/common/src/main/java/org/apache/omid/YAMLUtils.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.Resources;
+import org.apache.commons.beanutils.BeanUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+public class YAMLUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YAMLUtils.class);
+
+ public void loadSettings(String resourcePath, String defaultResourcePath, Object bean) {
+ try {
+ Map properties = loadSettings(resourcePath, defaultResourcePath);
+ BeanUtils.populate(bean, properties);
+ } catch (IllegalAccessException | InvocationTargetException | IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public void loadSettings(String resourcePath, Object bean) {
+ try {
+ Map properties = loadSettings(null, resourcePath);
+ BeanUtils.populate(bean, properties);
+ } catch (IllegalAccessException | InvocationTargetException | IOException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map loadSettings(String resourcePath, String defaultResourcePath) throws IOException {
+ Map defaultSetting = loadAsMap(defaultResourcePath);
+ Preconditions.checkState(defaultSetting.size() > 0, String.format("Failed to load file '%s' from classpath", defaultResourcePath));
+ if (resourcePath != null) {
+ Map userSetting = loadAsMap(resourcePath);
+ defaultSetting.putAll(userSetting);
+ }
+ return defaultSetting;
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map loadAsMap(String path) throws IOException {
+ try {
+ String content = Resources.toString(Resources.getResource(path), Charset.forName("UTF-8"));
+ LOG.debug("Loaded resource file '{}'\n{}", path, content);
+ Map settings = new Yaml().loadAs(content, Map.class);
+ if (settings == null) {
+ settings = new HashMap(0);
+ }
+ return settings;
+ } catch (IllegalArgumentException e) {
+ return new HashMap();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/main/java/org/apache/omid/zk/ZKUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/omid/zk/ZKUtils.java b/common/src/main/java/org/apache/omid/zk/ZKUtils.java
new file mode 100644
index 0000000..24c0ee1
--- /dev/null
+++ b/common/src/main/java/org/apache/omid/zk/ZKUtils.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.zk;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public class ZKUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKUtils.class);
+
+ public static CuratorFramework initZKClient(String zkCluster, String namespace, int zkConnectionTimeoutInSec)
+ throws IOException, InterruptedException {
+
+ LOG.info("Creating Zookeeper Client connecting to {}", zkCluster);
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+ CuratorFramework zkClient = CuratorFrameworkFactory.builder()
+ .namespace(namespace)
+ .connectString(zkCluster)
+ .retryPolicy(retryPolicy)
+ .build();
+
+ zkClient.start();
+ if (zkClient.blockUntilConnected(zkConnectionTimeoutInSec, TimeUnit.SECONDS)) {
+ LOG.info("Connected to ZK cluster '{}', client in state: [{}]", zkCluster, zkClient.getState());
+ } else {
+ throw new IOException(String.format("Can't contact ZK cluster '%s' after 10 seconds", zkCluster));
+ }
+
+ return zkClient;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/main/proto/TSOProto.proto
----------------------------------------------------------------------
diff --git a/common/src/main/proto/TSOProto.proto b/common/src/main/proto/TSOProto.proto
index 23762c5..43987d8 100644
--- a/common/src/main/proto/TSOProto.proto
+++ b/common/src/main/proto/TSOProto.proto
@@ -16,7 +16,7 @@
// limitations under the License.
//
-option java_package = "com.yahoo.omid.proto";
+option java_package = "org.apache.omid.proto";
option optimize_for = SPEED;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java b/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
deleted file mode 100644
index 172e898..0000000
--- a/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid;
-
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-public class YAMLUtilsTest {
-
- @Test
- public void testLoadDefaultSettings_setToBean() throws Exception {
- Map map = new HashMap();
- new YAMLUtils().loadSettings("test.yml", "default-test.yml", map);
- Assert.assertNotNull(map);
- Assert.assertEquals(map.get("prop1"), 11);
- Assert.assertEquals(map.get("prop2"), "22");
- Assert.assertEquals(map.get("prop3"), 3);
- }
-
- @Test
- public void testLoadDefaultSettings_setToBean2() throws Exception {
- Map map = new HashMap();
- new YAMLUtils().loadSettings("test.yml", map);
- Assert.assertNotNull(map);
- Assert.assertEquals(map.get("prop1"), 11);
- Assert.assertEquals(map.get("prop2"), "22");
- Assert.assertEquals(map.size(), 2);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/common/src/test/java/org/apache/omid/YAMLUtilsTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/omid/YAMLUtilsTest.java b/common/src/test/java/org/apache/omid/YAMLUtilsTest.java
new file mode 100644
index 0000000..b296922
--- /dev/null
+++ b/common/src/test/java/org/apache/omid/YAMLUtilsTest.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class YAMLUtilsTest {
+
+ @Test
+ public void testLoadDefaultSettings_setToBean() throws Exception {
+ Map map = new HashMap();
+ new YAMLUtils().loadSettings("test.yml", "default-test.yml", map);
+ Assert.assertNotNull(map);
+ Assert.assertEquals(map.get("prop1"), 11);
+ Assert.assertEquals(map.get("prop2"), "22");
+ Assert.assertEquals(map.get("prop3"), 3);
+ }
+
+ @Test
+ public void testLoadDefaultSettings_setToBean2() throws Exception {
+ Map map = new HashMap();
+ new YAMLUtils().loadSettings("test.yml", map);
+ Assert.assertNotNull(map);
+ Assert.assertEquals(map.get("prop1"), 11);
+ Assert.assertEquals(map.get("prop2"), "22");
+ Assert.assertEquals(map.size(), 2);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 592fe28..192f414 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>omid</artifactId>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -18,13 +18,13 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>codahale-metrics</artifactId>
<version>${project.version}</version>
</dependency>
@@ -112,7 +112,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -123,7 +123,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/run.sh
----------------------------------------------------------------------
diff --git a/examples/run.sh b/examples/run.sh
index c3ea2e9..6785919 100755
--- a/examples/run.sh
+++ b/examples/run.sh
@@ -84,13 +84,13 @@ USER_OPTION=$1
shift
case ${USER_OPTION} in
basic)
- java -cp $KLASSPATH com.yahoo.omid.examples.BasicExample "$@"
+ java -cp $KLASSPATH org.apache.omid.examples.BasicExample "$@"
;;
si)
- java -cp $KLASSPATH com.yahoo.omid.examples.SnapshotIsolationExample "$@"
+ java -cp $KLASSPATH org.apache.omid.examples.SnapshotIsolationExample "$@"
;;
conf)
- java -cp $KLASSPATH com.yahoo.omid.examples.ConfigurationExample "$@"
+ java -cp $KLASSPATH org.apache.omid.examples.ConfigurationExample "$@"
;;
*)
show_help
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java b/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
deleted file mode 100644
index 44d9646..0000000
--- a/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.examples;
-
-import com.yahoo.omid.transaction.HBaseTransactionManager;
-import com.yahoo.omid.transaction.TTable;
-import com.yahoo.omid.transaction.Transaction;
-import com.yahoo.omid.transaction.TransactionManager;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * ****************************************************************************************************
- *
- * Example code which demonstrates an atomic write into two different rows in HBase
- *
- * ****************************************************************************************************
- *
- * After building the package with 'mvn clean package' find the resulting examples-<version>-bin.tar.gz file in the
- * 'examples/target' folder. Copy it to the target host and expand with 'tar -zxvf examples-<version>-bin.tar.gz'.
- *
- * Make sure that 'hbase-site.xml' and 'core-site.xml' are either in classpath (see run.sh) or explicitly referenced in
- * configuration file. If a secure HBase deployment is needed, make sure to specify the principal (user) and keytab file.
- *
- * The example requires a user table to perform transactional read/write operations. A table is already specified in
- * the default configuration, and can be created with the following command using the 'hbase shell':
- *
- * <pre>
- * create 'MY_TX_TABLE', {NAME => 'MY_CF', VERSIONS => '2147483647', TTL => '2147483647'}
- * </pre>
- *
- * Make sure that the principal/user has RW permissions for the given table using also the 'hbase shell':
- * <pre>
- * grant '<principal/user>', 'RW', 'MY_TX_TABLE'
- * </pre>
- *
- * Alternatively, a table with a column family already created can be used by specifying the table name and column
- * family identifiers using the command line arguments (see details also in 'run.sh') If a table namespace is required,
- * specify it like this: 'namespace:table_name'
- *
- * Finally, run the example using the 'run.sh' script without arguments or specifying the necessary configuration
- * parameters.
- */
-public class BasicExample {
-
- private static final Logger LOG = LoggerFactory.getLogger(BasicExample.class);
-
- public static void main(String[] args) throws Exception {
-
- LOG.info("Parsing command line arguments");
- String userTableName = "MY_TX_TABLE";
- if (args != null && args.length > 0 && StringUtils.isNotEmpty(args[0])) {
- userTableName = args[0];
- }
- byte[] family = Bytes.toBytes("MY_CF");
- if (args != null && args.length > 1 && StringUtils.isNotEmpty(args[1])) {
- family = Bytes.toBytes(args[1]);
- }
- LOG.info("Table '{}', column family '{}'", userTableName, Bytes.toString(family));
-
- byte[] exampleRow1 = Bytes.toBytes("EXAMPLE_ROW1");
- byte[] exampleRow2 = Bytes.toBytes("EXAMPLE_ROW2");
- byte[] qualifier = Bytes.toBytes("MY_Q");
- byte[] dataValue1 = Bytes.toBytes("val1");
- byte[] dataValue2 = Bytes.toBytes("val2");
-
- LOG.info("Creating access to Omid Transaction Manager & Transactional Table '{}'", userTableName);
- try (TransactionManager tm = HBaseTransactionManager.newInstance();
- TTable txTable = new TTable(userTableName))
- {
- Transaction tx = tm.begin();
- LOG.info("Transaction {} STARTED", tx);
-
- Put row1 = new Put(exampleRow1);
- row1.add(family, qualifier, dataValue1);
- txTable.put(tx, row1);
- LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
- tx, userTableName, Bytes.toString(exampleRow1), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue1));
-
- Put row2 = new Put(exampleRow2);
- row2.add(family, qualifier, dataValue2);
- txTable.put(tx, row2);
- LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
- tx, userTableName, Bytes.toString(exampleRow2), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue2));
-
- tm.commit(tx);
- LOG.info("Transaction {} COMMITTED", tx);
- }
-
- }
-
-}
[06/52] [abbrv] incubator-omid git commit: [ci skip]prepare release
omid-0.8.1.35
Posted by ik...@apache.org.
[ci skip]prepare release omid-0.8.1.35
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/a3887f25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/a3887f25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/a3887f25
Branch: refs/heads/master
Commit: a3887f2504913cfb6f4cb0017b20fcfaf686d86c
Parents: a02d74a
Author: Omid CI <om...@yahoo-inc.com>
Authored: Wed Apr 13 21:10:06 2016 +0000
Committer: Omid CI <om...@yahoo-inc.com>
Committed: Wed Apr 13 21:10:06 2016 +0000
----------------------------------------------------------------------
benchmarks/pom.xml | 2 +-
codahale-metrics/pom.xml | 2 +-
commit-table/pom.xml | 2 +-
common/pom.xml | 2 +-
examples/pom.xml | 2 +-
hbase-client/pom.xml | 2 +-
hbase-commit-table/pom.xml | 2 +-
hbase-common/pom.xml | 2 +-
hbase-coprocessor/pom.xml | 2 +-
hbase-shims/hbase-0/pom.xml | 2 +-
hbase-shims/hbase-1/pom.xml | 2 +-
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 2 +-
metrics/pom.xml | 2 +-
pom.xml | 4 ++--
statemachine/pom.xml | 2 +-
timestamp-storage/pom.xml | 2 +-
transaction-client/pom.xml | 2 +-
tso-server/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index 315ee19..d5a3c72 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>benchmarks</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index fac7ccc..362ae0d 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index 30b1d5c..b069d15 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 1d7d1cc..8ae6dc9 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index ae20894..d0b027b 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 51b4b80..4238091 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index 3420d33..63e2a76 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase-commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index ea5400f..2adfd62 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index cc100fb..f90e4cc 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase-coprocessor</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index fa1264c..5be179d 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase0-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index fd9f869..4e556ac 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase1-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index b0ef51c..8fc98e5 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>omid-shims-aggregator</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index 5910986..6aba19f 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>hbase-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index b1c0359..e481c73 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>metrics</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 10e4438..402423e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
<licenses>
<license>
@@ -46,7 +46,7 @@
<connection>scm:git:git://github.com/yahoo/omid.git</connection>
<developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
<url>https://github.com/yahoo/omid</url>
- <tag>omid-0.8.1.21</tag>
+ <tag>omid-0.8.1.35</tag>
</scm>
<distributionManagement>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index c7f9d7f..047c265 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<name>State Machine</name>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index b25429d..a512f12 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>timestamp-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index 5b4950b..f398046 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>transaction-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/a3887f25/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index 0342192..22a7e3b 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35-SNAPSHOT</version>
+ <version>0.8.1.35</version>
</parent>
<artifactId>tso-server</artifactId>
[34/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java
new file mode 100644
index 0000000..03599f7
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java
@@ -0,0 +1,508 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.omid.committable.CommitTable;
+
+import org.apache.omid.metrics.NullMetricsProvider;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.mockito.Matchers;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.omid.transaction.CellUtils.hasCell;
+import static org.apache.omid.transaction.CellUtils.hasShadowCell;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestShadowCells extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestShadowCells.class);
+
+ private static final String TSO_SERVER_HOST = "localhost";
+ private static final int TSO_SERVER_PORT = 1234;
+
+ private static final String TEST_TABLE = "test";
+ private static final String TEST_FAMILY = "data";
+
+ static final byte[] row = Bytes.toBytes("test-sc");
+ private static final byte[] row1 = Bytes.toBytes("test-sc1");
+ private static final byte[] row2 = Bytes.toBytes("test-sc2");
+ private static final byte[] row3 = Bytes.toBytes("test-sc3");
+ static final byte[] family = Bytes.toBytes(TEST_FAMILY);
+ private static final byte[] qualifier = Bytes.toBytes("testdata");
+ private static final byte[] data1 = Bytes.toBytes("testWrite-1");
+
+
+ @Test(timeOut = 60_000)
+ public void testShadowCellsBasics(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+
+ // Test shadow cells are created properly
+ Put put = new Put(row);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+
+ // Before commit test that only the cell is there
+ assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Cell should be there");
+ assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Shadow cell shouldn't be there");
+
+ tm.commit(t1);
+
+ // After commit test that both cell and shadow cell are there
+ assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Cell should be there");
+ assertTrue(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Shadow cell should be there");
+
+ // Test that we can make a valid read after adding a shadow cell without hitting the commit table
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ TransactionManager tm2 = HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .commitTableClient(commitTableClient)
+ .build();
+
+ Transaction t2 = tm2.begin();
+ Get get = new Get(row);
+ get.addColumn(family, qualifier);
+
+ Result getResult = table.get(t2, get);
+ assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
+ verify(commitTableClient, never()).getCommitTimestamp(anyLong());
+ }
+
+ @Test(timeOut = 60_000)
+ public void testCrashingAfterCommitDoesNotWriteShadowCells(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ PostCommitActions syncPostCommitter = spy(
+ new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .postCommitter(syncPostCommitter)
+ .commitTableClient(commitTableClient)
+ .build());
+
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+
+ // Test shadow cell are created properly
+ Put put = new Put(row);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+ try {
+ tm.commit(t1);
+ } catch (Exception e) { // (*) crash
+ // Do nothing
+ }
+
+ // After commit with the emulated crash, test that only the cell is there
+ assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Cell should be there");
+ assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Shadow cell should not be there");
+
+ Transaction t2 = tm.begin();
+ Get get = new Get(row);
+ get.addColumn(family, qualifier);
+
+ Result getResult = table.get(t2, get);
+ assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Shadow cell should not be there");
+ verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
+ }
+
+ @Test(timeOut = 60_000)
+ public void testShadowCellIsHealedAfterCommitCrash(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ PostCommitActions syncPostCommitter = spy(
+ new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .postCommitter(syncPostCommitter)
+ .commitTableClient(commitTableClient)
+ .build());
+
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+
+ // Test shadow cell are created properly
+ Put put = new Put(row);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+ try {
+ tm.commit(t1);
+ } catch (Exception e) { // (*) Crash
+ // Do nothing
+ }
+
+ assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Cell should be there");
+ assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Shadow cell should not be there");
+
+ Transaction t2 = tm.begin();
+ Get get = new Get(row);
+ get.addColumn(family, qualifier);
+
+ // This get should heal the shadow cell
+ Result getResult = table.get(t2, get);
+ assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
+ verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
+
+ assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Cell should be there");
+ assertTrue(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Shadow cell should be there after being healed");
+
+ // As the shadow cell is healed, this get shouldn't have to hit the storage,
+ // so the number of invocations to commitTableClient.getCommitTimestamp()
+ // should remain the same
+ getResult = table.get(t2, get);
+ assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
+ verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
+ }
+
+ @Test(timeOut = 60_000)
+ public void testTransactionNeverCompletesWhenAnExceptionIsThrownUpdatingShadowCells(ITestContext context)
+ throws Exception {
+
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ PostCommitActions syncPostCommitter = spy(
+ new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .postCommitter(syncPostCommitter)
+ .commitTableClient(commitTableClient)
+ .build());
+
+ final TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction tx = (HBaseTransaction) tm.begin();
+
+ Put put = new Put(row);
+ put.add(family, qualifier, data1);
+ table.put(tx, put);
+
+ // This line emulates an error accessing the target table by disabling it
+ doAnswer(new Answer<ListenableFuture<Void>>() {
+ @Override
+ public ListenableFuture<Void> answer(InvocationOnMock invocation) throws Throwable {
+ table.flushCommits();
+ HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
+ admin.disableTable(table.getTableName());
+ return (ListenableFuture<Void>) invocation.callRealMethod();
+ }
+ }).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ // When committing, an IOException should be thrown in syncPostCommitter.updateShadowCells() and placed in the
+ // future as a TransactionManagerException. However, the exception is never retrieved in the
+ // AbstractTransactionManager as the future is never checked.
+ // This requires to set the HConstants.HBASE_CLIENT_RETRIES_NUMBER in the HBase config to a finite number:
+ // e.g -> hbaseConf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 3); Otherwise it will get stuck in tm.commit();
+
+ tm.commit(tx); // Tx effectively commits but the post Commit Actions failed when updating the shadow cells
+
+ // Re-enable table to allow the required checks below
+ HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
+ admin.enableTable(table.getTableName());
+
+ // 1) check that shadow cell is not created...
+ assertTrue(hasCell(row, family, qualifier, tx.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Cell should be there");
+ assertFalse(hasShadowCell(row, family, qualifier, tx.getStartTimestamp(), new TTableCellGetterAdapter(table)),
+ "Shadow cell should not be there");
+ // 2) and thus, completeTransaction() was never called on the commit table...
+ verify(commitTableClient, times(0)).completeTransaction(anyLong());
+ // 3) ...and commit value still in commit table
+ assertTrue(commitTableClient.getCommitTimestamp(tx.getStartTimestamp()).get().isPresent());
+
+ }
+
+ @Test(timeOut = 60_000)
+ public void testRaceConditionBetweenReaderAndWriterThreads(final ITestContext context) throws Exception {
+ final CountDownLatch readAfterCommit = new CountDownLatch(1);
+ final CountDownLatch postCommitBegin = new CountDownLatch(1);
+ final CountDownLatch postCommitEnd = new CountDownLatch(1);
+
+ final AtomicBoolean readFailed = new AtomicBoolean(false);
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
+ AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
+
+ doAnswer(new Answer<ListenableFuture<Void>>() {
+ @Override
+ public ListenableFuture<Void> answer(InvocationOnMock invocation) throws Throwable {
+ LOG.info("Releasing readAfterCommit barrier");
+ readAfterCommit.countDown();
+ LOG.info("Waiting postCommitBegin barrier");
+ postCommitBegin.await();
+ ListenableFuture<Void> result = (ListenableFuture<Void>) invocation.callRealMethod();
+ LOG.info("Releasing postCommitEnd barrier");
+ postCommitEnd.countDown();
+ return result;
+ }
+ }).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ // Start transaction on write thread
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ final HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+
+ // Start read thread
+ Thread readThread = new Thread("Read Thread") {
+ @Override
+ public void run() {
+ LOG.info("Waiting readAfterCommit barrier");
+ try {
+ readAfterCommit.await();
+ final TTable table = spy(new TTable(hbaseConf, TEST_TABLE));
+ doAnswer(new Answer<List<KeyValue>>() {
+ @SuppressWarnings("unchecked")
+ @Override
+ public List<KeyValue> answer(InvocationOnMock invocation) throws Throwable {
+ LOG.info("Release postCommitBegin barrier");
+ postCommitBegin.countDown();
+ LOG.info("Waiting postCommitEnd barrier");
+ postCommitEnd.await();
+ return (List<KeyValue>) invocation.callRealMethod();
+ }
+ }).when(table).filterCellsForSnapshot(Matchers.<List<Cell>>any(),
+ any(HBaseTransaction.class), anyInt());
+
+ TransactionManager tm = newTransactionManager(context);
+ if (hasShadowCell(row,
+ family,
+ qualifier,
+ t1.getStartTimestamp(),
+ new TTableCellGetterAdapter(table))) {
+ readFailed.set(true);
+ }
+
+ Transaction t = tm.begin();
+ Get get = new Get(row);
+ get.addColumn(family, qualifier);
+
+ Result getResult = table.get(t, get);
+ Cell cell = getResult.getColumnLatestCell(family, qualifier);
+ if (!Arrays.equals(data1, CellUtil.cloneValue(cell))
+ || !hasShadowCell(row,
+ family,
+ qualifier,
+ cell.getTimestamp(),
+ new TTableCellGetterAdapter(table))) {
+ readFailed.set(true);
+ } else {
+ LOG.info("Read succeeded");
+ }
+ } catch (Throwable e) {
+ readFailed.set(true);
+ LOG.error("Error whilst reading", e);
+ }
+ }
+ };
+ readThread.start();
+
+ // Write data
+ Put put = new Put(row);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+ tm.commit(t1);
+
+ readThread.join();
+
+ assertFalse(readFailed.get(), "Read should have succeeded");
+
+ }
+
+ // TODO: After removing the legacy shadow cell suffix, maybe we should mix the assertions in this test with
+ // the ones in the previous tests in a further commit
+
+ /**
+ * Test that the new client can read shadow cells written by the old client.
+ */
+ @Test
+ public void testGetOldShadowCells(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+ HTableInterface htable = table.getHTable();
+
+ // Test shadow cell are created properly
+ HBaseTransaction t1 = (HBaseTransaction) tm.begin();
+ Put put = new Put(row1);
+ put.add(family, qualifier, data1);
+ table.put(t1, put);
+ tm.commit(t1);
+
+ HBaseTransaction t2 = (HBaseTransaction) tm.begin();
+ put = new Put(row2);
+ put.add(family, qualifier, data1);
+ table.put(t2, put);
+ tm.commit(t2);
+
+ HBaseTransaction t3 = (HBaseTransaction) tm.begin();
+ put = new Put(row3);
+ put.add(family, qualifier, data1);
+ table.put(t3, put);
+ tm.commit(t3);
+
+ // ensure that transaction is no longer in commit table
+ // the only place that should have the mapping is the shadow cells
+ CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
+ Optional<CommitTable.CommitTimestamp> ct1 = commitTableClient.getCommitTimestamp(t1.getStartTimestamp()).get();
+ Optional<CommitTable.CommitTimestamp> ct2 = commitTableClient.getCommitTimestamp(t2.getStartTimestamp()).get();
+ Optional<CommitTable.CommitTimestamp> ct3 = commitTableClient.getCommitTimestamp(t3.getStartTimestamp()).get();
+ assertFalse(ct1.isPresent(), "Shouldn't exist in commit table");
+ assertFalse(ct2.isPresent(), "Shouldn't exist in commit table");
+ assertFalse(ct3.isPresent(), "Shouldn't exist in commit table");
+
+ // delete new shadow cell
+ Delete del = new Delete(row2);
+ del.deleteColumn(family, CellUtils.addShadowCellSuffix(qualifier));
+ htable.delete(del);
+ htable.flushCommits();
+
+ // verify that we can't read now (since shadow cell is missing)
+ Transaction t4 = tm.begin();
+ Get get = new Get(row2);
+ get.addColumn(family, qualifier);
+
+ Result getResult = table.get(t4, get);
+ assertTrue(getResult.isEmpty(), "Should have nothing");
+
+ Transaction t5 = tm.begin();
+ Scan s = new Scan();
+ ResultScanner scanner = table.getScanner(t5, s);
+ Result result1 = scanner.next();
+ Result result2 = scanner.next();
+ Result result3 = scanner.next();
+ scanner.close();
+
+ assertNull(result3);
+ assertTrue(Arrays.equals(result1.getRow(), row1), "Should have first row");
+ assertTrue(Arrays.equals(result2.getRow(), row3), "Should have third row");
+ assertTrue(result1.containsColumn(family, qualifier), "Should have column family");
+ assertTrue(result2.containsColumn(family, qualifier), "Should have column family");
+
+ // now add in the previous legacy shadow cell for that row
+ put = new Put(row2);
+ put.add(family,
+ addLegacyShadowCellSuffix(qualifier),
+ t2.getStartTimestamp(),
+ Bytes.toBytes(t2.getCommitTimestamp()));
+ htable.put(put);
+
+ // we should NOT be able to read that row now, even though
+ // it has a legacy shadow cell
+ Transaction t6 = tm.begin();
+ get = new Get(row2);
+ get.addColumn(family, qualifier);
+
+ getResult = table.get(t6, get);
+ assertFalse(getResult.containsColumn(family, qualifier), "Should NOT have column");
+
+ Transaction t7 = tm.begin();
+ s = new Scan();
+ scanner = table.getScanner(t7, s);
+ result1 = scanner.next();
+ result2 = scanner.next();
+ result3 = scanner.next();
+ scanner.close();
+
+ assertNull(result3, "There should only be 2 rows");
+ assertTrue(Arrays.equals(result1.getRow(), row1), "Should have first row");
+ assertTrue(Arrays.equals(result2.getRow(), row3), "Should have third row");
+ assertTrue(result1.containsColumn(family, qualifier), "Should have column family");
+ assertTrue(result2.containsColumn(family, qualifier), "Should have column family");
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private static final byte[] LEGACY_SHADOW_CELL_SUFFIX = ":OMID_CTS".getBytes(Charsets.UTF_8);
+
+ private static byte[] addLegacyShadowCellSuffix(byte[] qualifier) {
+ return com.google.common.primitives.Bytes.concat(qualifier, LEGACY_SHADOW_CELL_SUFFIX);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestSingleColumnFamily.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestSingleColumnFamily.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestSingleColumnFamily.java
new file mode 100644
index 0000000..c76085f
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestSingleColumnFamily.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestSingleColumnFamily extends OmidTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(TestSingleColumnFamily.class);
+
+
+ @Test
+ public void testSingleColumnFamily(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable table1 = new TTable(hbaseConf, TEST_TABLE);
+ int num = 10;
+ Transaction t = tm.begin();
+ for (int j = 0; j < num; j++) {
+ byte[] data = Bytes.toBytes(j);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value1"), data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2"), data);
+ table1.put(t, put);
+ }
+ //tm.tryCommit(t);
+ //t=tm.beginTransaction(); //Visible if in a different transaction
+ Scan s = new Scan();
+ ResultScanner res = table1.getScanner(t, s);
+ Result rr;
+ int count = 0;
+ while ((rr = res.next()) != null) {
+ int tmp1 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value1")));
+ int tmp2 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2")));
+ LOG.info("RES:" + tmp1 + ";" + tmp2);
+ count++;
+ }
+ assertTrue("Can't see puts. I should see "
+ + num + " but I see " + count
+ , num == count);
+
+ tm.commit(t);
+ t = tm.begin();
+
+ for (int j = 0; j < num / 2; j++) {
+ byte[] data = Bytes.toBytes(j);
+ byte[] ndata = Bytes.toBytes(j * 10);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2"), ndata);
+ table1.put(t, put);
+ }
+ tm.commit(t);
+ t = tm.begin();
+ s = new Scan();
+ res = table1.getScanner(t, s);
+ count = 0;
+ int modified = 0, notmodified = 0;
+ while ((rr = res.next()) != null) {
+ int tmp1 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value1")));
+ int tmp2 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2")));
+ LOG.info("RES:" + tmp1 + ";" + tmp2);
+ count++;
+
+ if (tmp2 == Bytes.toInt(rr.getRow()) * 10) {
+ modified++;
+ } else {
+ notmodified++;
+ }
+ if (count == 8) {
+ LOG.debug("stop");
+ }
+ }
+ assertTrue("Can't see puts. I should see "
+ + num + " but I see " + count
+ , num == count);
+ assertTrue("Half of rows should equal row id, half not ("
+ + modified + ", " + notmodified + ")"
+ , modified == notmodified && notmodified == (num / 2));
+
+ tm.commit(t);
+ LOG.info("End commiting");
+ table1.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestTSOModule.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestTSOModule.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestTSOModule.java
new file mode 100644
index 0000000..601c6ca
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestTSOModule.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorage;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.apache.omid.tso.DisruptorModule;
+import org.apache.omid.tso.MockPanicker;
+import org.apache.omid.tso.NetworkInterfaceUtils;
+import org.apache.omid.tso.Panicker;
+import org.apache.omid.tso.PausableTimestampOracle;
+import org.apache.omid.tso.TSOChannelHandler;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.omid.tso.TSOStateManager;
+import org.apache.omid.tso.TSOStateManagerImpl;
+import org.apache.omid.tso.TimestampOracle;
+import org.apache.hadoop.conf.Configuration;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+class TestTSOModule extends AbstractModule {
+
+ private final Configuration hBaseConfig;
+ private final TSOServerConfig config;
+
+ TestTSOModule(Configuration hBaseConfig, TSOServerConfig config) {
+ this.hBaseConfig = hBaseConfig;
+ this.config = config;
+ }
+
+ @Override
+ protected void configure() {
+
+ bind(TSOChannelHandler.class).in(Singleton.class);
+
+ bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
+
+ bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class);
+ bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class);
+ bind(TimestampOracle.class).to(PausableTimestampOracle.class).in(Singleton.class);
+ bind(Panicker.class).to(MockPanicker.class).in(Singleton.class);
+
+ // Disruptor setup
+ install(new DisruptorModule());
+
+ // LeaseManagement setup
+ install(config.getLeaseModule());
+ }
+
+ @Provides
+ Configuration provideHBaseConfig() {
+ return hBaseConfig;
+ }
+
+ @Provides
+ TSOServerConfig provideTSOServerConfig() {
+ return config;
+ }
+
+ @Provides
+ @Singleton
+ MetricsRegistry provideMetricsRegistry() {
+ return new NullMetricsProvider();
+ }
+
+ @Provides
+ @Named(TSO_HOST_AND_PORT_KEY)
+ String provideTSOHostAndPort() throws SocketException, UnknownHostException {
+ return NetworkInterfaceUtils.getTSOHostAndPort(config);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestTTableBehaviour.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestTTableBehaviour.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestTTableBehaviour.java
new file mode 100644
index 0000000..9906e4b
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestTTableBehaviour.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Charsets;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.mockito.Mockito;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.fail;
+
+@Test(groups = "noHBase")
+public class TestTTableBehaviour {
+
+ private byte[] row = Bytes.toBytes("1row");
+ private byte[] famName = Bytes.toBytes("tf");
+ private byte[] colName = Bytes.toBytes("tc");
+ private byte[] dataValue = Bytes.toBytes("test-data");
+
+ @Test(timeOut = 10_000)
+ public void testUserOperationsDontAllowTimestampSpecification() throws Exception {
+
+ // Component under test
+ TTable tt = new TTable(Mockito.mock(HTableInterface.class), Mockito.mock(HTableInterface.class));
+
+ long randomTimestampValue = Bytes.toLong("deadbeef".getBytes());
+
+ Transaction tx = Mockito.mock(Transaction.class);
+
+ // Test put fails when a timestamp is specified in the put
+ Put put = new Put(row, randomTimestampValue);
+ put.add(famName, colName, dataValue);
+ try {
+ tt.put(tx, put);
+ fail("Should have thrown an IllegalArgumentException due to timestamp specification");
+ } catch (IllegalArgumentException e) {
+ // Continue
+ }
+
+ // Test put fails when a timestamp is specified in a qualifier
+ put = new Put(row);
+ put.add(famName, colName, randomTimestampValue, dataValue);
+ try {
+ tt.put(tx, put);
+ fail("Should have thrown an IllegalArgumentException due to timestamp specification");
+ } catch (IllegalArgumentException e) {
+ // Continue
+ }
+
+ // Test that get fails when a timestamp is specified
+ Get get = new Get(row);
+ get.setTimeStamp(randomTimestampValue);
+ try {
+ tt.get(tx, get);
+ fail("Should have thrown an IllegalArgumentException due to timestamp specification");
+ } catch (IllegalArgumentException e) {
+ // Continue
+ }
+
+ // Test scan fails when a timerange is specified
+ Scan scan = new Scan(get);
+ try {
+ tt.getScanner(tx, scan);
+ fail("Should have thrown an IllegalArgumentException due to timestamp specification");
+ } catch (IllegalArgumentException e) {
+ // Continue
+ }
+
+ // Test delete fails when a timestamp is specified
+ Delete delete = new Delete(row);
+ delete.setTimestamp(randomTimestampValue);
+ try {
+ tt.delete(tx, delete);
+ fail("Should have thrown an IllegalArgumentException due to timestamp specification");
+ } catch (IllegalArgumentException e) {
+ // Continue
+ }
+
+ // Test delete fails when a timestamp is specified in a qualifier
+ delete = new Delete(row);
+ delete.deleteColumn(famName, colName, randomTimestampValue);
+ try {
+ tt.delete(tx, delete);
+ fail("Should have thrown an IllegalArgumentException due to timestamp specification");
+ } catch (IllegalArgumentException e) {
+ // Continue
+ }
+
+ }
+
+ /**
+ * Test that we cannot use reserved names for shadow cell identifiers as qualifiers in user operations
+ */
+ @Test(timeOut = 10_000)
+ public void testReservedNamesForShadowCellsCanNotBeUsedAsQualifiersInUserOperations() throws Exception {
+ byte[] nonValidQualifier1 = "blahblah\u0080".getBytes(Charsets.UTF_8);
+ byte[] validQualifierIncludingOldShadowCellSuffix = "blahblah:OMID_CTS".getBytes(Charsets.UTF_8);
+
+ TTable table = new TTable(Mockito.mock(HTableInterface.class), Mockito.mock(HTableInterface.class));
+
+ HBaseTransaction t1 = Mockito.mock(HBaseTransaction.class);
+ Put put = new Put(row);
+ put.add(famName, nonValidQualifier1, dataValue);
+ try {
+ table.put(t1, put);
+ fail("Shouldn't be able to put this");
+ } catch (IllegalArgumentException iae) {
+ // correct
+ }
+ Delete del = new Delete(row);
+ del.deleteColumn(famName, nonValidQualifier1);
+ try {
+ table.delete(t1, del);
+ fail("Shouldn't be able to delete this");
+ } catch (IllegalArgumentException iae) {
+ // correct
+ }
+
+ put = new Put(row);
+ put.add(famName, validQualifierIncludingOldShadowCellSuffix, dataValue);
+ try {
+ table.put(t1, put);
+ } catch (IllegalArgumentException iae) {
+ fail("Qualifier shouldn't be rejected anymore");
+ }
+ del = new Delete(row);
+ del.deleteColumn(famName, validQualifierIncludingOldShadowCellSuffix);
+ try {
+ table.delete(t1, del);
+ } catch (IllegalArgumentException iae) {
+ fail("Qualifier shouldn't be rejected anymore");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionCleanup.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionCleanup.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionCleanup.java
new file mode 100644
index 0000000..6b66710
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionCleanup.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.tso.client.AbortException;
+import org.apache.omid.tso.client.ForwardingTSOFuture;
+import org.apache.omid.tso.client.TSOClient;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.mockito.Matchers.anySetOf;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.testng.AssertJUnit.assertEquals;
+
+@Test(groups = "sharedHBase")
+public class TestTransactionCleanup extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTransactionCleanup.class);
+
+ private static final long START_TS = 1L;
+
+ private byte[] row = Bytes.toBytes("row");
+ private byte[] family = Bytes.toBytes(TEST_FAMILY);
+ private byte[] qual = Bytes.toBytes("qual");
+ private byte[] data = Bytes.toBytes("data");
+
+ // NOTE: This test is maybe redundant with runTestCleanupAfterConflict()
+ // and testCleanupWithDeleteRow() tests in TestTransactionCleanup class.
+ // Code in TestTransactionCleanup is a little more difficult to follow,
+ // lacks some assertions and includes some magic numbers, so we should
+ // try to review and improve the tests in these two classes in a further
+ // commit.
+ @Test
+ public void testTransactionIsCleanedUpAfterBeingAborted(ITestContext context) throws Exception {
+
+ final int ROWS_MODIFIED = 1;
+
+ // Prepare the mocking results
+ SettableFuture<Long> startTSF = SettableFuture.create();
+ startTSF.set(START_TS);
+ ForwardingTSOFuture<Long> stFF = new ForwardingTSOFuture<>(startTSF);
+
+ SettableFuture<Long> abortingF = SettableFuture.create();
+ abortingF.setException(new AbortException());
+ ForwardingTSOFuture<Long> abortingFF = new ForwardingTSOFuture<>(abortingF);
+
+ // Mock the TSO Client setting the right method responses
+ TSOClient mockedTSOClient = mock(TSOClient.class);
+
+ doReturn(stFF)
+ .when(mockedTSOClient).getNewStartTimestamp();
+
+ doReturn(abortingFF)
+ .when(mockedTSOClient).commit(eq(START_TS), anySetOf(HBaseCellId.class));
+
+ try (TransactionManager tm = newTransactionManager(context, mockedTSOClient);
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Start a transaction and put some data in a column
+ Transaction tx = tm.begin();
+
+ Put put = new Put(row);
+ put.add(family, qual, data);
+ txTable.put(tx, put);
+
+ // Abort transaction when committing, so the cleanup
+ // process we want to test is triggered
+ try {
+ tm.commit(tx);
+ } catch (RollbackException e) {
+ // Expected
+ }
+
+ // So now we have to check that the Delete marker introduced by the
+ // cleanup process is there
+ Scan scan = new Scan(row);
+ scan.setRaw(true); // Raw scan to obtain the deleted cells
+ ResultScanner resultScanner = txTable.getHTable().getScanner(scan);
+ int resultCount = 0;
+ for (Result result : resultScanner) {
+ assertEquals(2, result.size()); // Size == 2, including the put and delete from cleanup
+ LOG.trace("Result {}", result);
+ // The last element of the qualifier should have the Delete marker
+ byte encodedType = result.getColumnLatestCell(family, qual).getTypeByte();
+ assertEquals(KeyValue.Type.Delete,
+ KeyValue.Type.codeToType(encodedType));
+ resultCount++;
+ }
+ assertEquals(ROWS_MODIFIED, resultCount);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java
new file mode 100644
index 0000000..229acee
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java
@@ -0,0 +1,308 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestTransactionConflict extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTransactionConflict.class);
+
+
+ @Test
+ public void runTestWriteWriteConflict(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ Transaction t2 = tm.begin();
+ LOG.info("Transaction created" + t2);
+
+ byte[] row = Bytes.toBytes("test-simple");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+
+ Put p2 = new Put(row);
+ p2.add(fam, col, data2);
+ tt.put(t2, p2);
+
+ tm.commit(t2);
+
+ try {
+ tm.commit(t1);
+ Assert.fail("Transaction should not commit successfully");
+ } catch (RollbackException e) {
+ }
+ }
+
+ @Test
+ public void runTestMultiTableConflict(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+ String table2 = TEST_TABLE + 2;
+ TableName table2Name = TableName.valueOf(table2);
+
+ HBaseAdmin admin = new HBaseAdmin(hbaseConf);
+
+ if (!admin.tableExists(table2)) {
+ HTableDescriptor desc = new HTableDescriptor(table2Name);
+ HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY);
+ datafam.setMaxVersions(Integer.MAX_VALUE);
+ desc.addFamily(datafam);
+
+ admin.createTable(desc);
+ }
+
+ if (admin.isTableDisabled(table2)) {
+ admin.enableTable(table2);
+ }
+ admin.close();
+
+ TTable tt2 = new TTable(hbaseConf, table2);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ Transaction t2 = tm.begin();
+ LOG.info("Transaction created" + t2);
+
+ byte[] row = Bytes.toBytes("test-simple");
+ byte[] row2 = Bytes.toBytes("test-simple2");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+ tt2.put(t1, p);
+
+ Put p2 = new Put(row);
+ p2.add(fam, col, data2);
+ tt.put(t2, p2);
+ p2 = new Put(row2);
+ p2.add(fam, col, data2);
+ tt2.put(t2, p2);
+
+ tm.commit(t2);
+
+ boolean aborted = false;
+ try {
+ tm.commit(t1);
+ assertTrue("Transaction commited successfully", false);
+ } catch (RollbackException e) {
+ aborted = true;
+ }
+ assertTrue("Transaction didn't raise exception", aborted);
+
+ ResultScanner rs = tt2.getHTable().getScanner(fam, col);
+
+ int count = 0;
+ Result r;
+ while ((r = rs.next()) != null) {
+ count += r.size();
+ }
+ assertEquals("Should have cell", 1, count);
+ }
+
+ @Test
+ public void runTestCleanupAfterConflict(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ Transaction t2 = tm.begin();
+ LOG.info("Transaction created" + t2);
+
+ byte[] row = Bytes.toBytes("test-simple");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+
+ Get g = new Get(row).setMaxVersions();
+ g.addColumn(fam, col);
+ Result r = tt.getHTable().get(g);
+ assertEquals("Unexpected size for read.", 1, r.size());
+ assertTrue("Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)),
+ Bytes.equals(data1, r.getValue(fam, col)));
+
+ Put p2 = new Put(row);
+ p2.add(fam, col, data2);
+ tt.put(t2, p2);
+
+ r = tt.getHTable().get(g);
+ assertEquals("Unexpected size for read.", 2, r.size());
+ r = tt.get(t2, g);
+ assertEquals("Unexpected size for read.", 1, r.size());
+ assertTrue("Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)),
+ Bytes.equals(data2, r.getValue(fam, col)));
+
+ tm.commit(t1);
+
+ boolean aborted = false;
+ try {
+ tm.commit(t2);
+ assertTrue("Transaction commited successfully", false);
+ } catch (RollbackException e) {
+ aborted = true;
+ }
+ assertTrue("Transaction didn't raise exception", aborted);
+
+ r = tt.getHTable().get(g);
+ assertEquals("Unexpected size for read.", 1, r.size());
+ assertTrue("Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)),
+ Bytes.equals(data1, r.getValue(fam, col)));
+ }
+
+ @Test
+ public void testCleanupWithDeleteRow(ITestContext context) throws Exception {
+ try {
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ int rowcount = 10;
+ int count = 0;
+
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ byte[] modrow = Bytes.toBytes("test-del" + 3);
+ for (int i = 0; i < rowcount; i++) {
+ byte[] row = Bytes.toBytes("test-del" + i);
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+ }
+ tm.commit(t1);
+
+ Transaction t2 = tm.begin();
+ LOG.info("Transaction created " + t2);
+ Delete d = new Delete(modrow);
+ tt.delete(t2, d);
+
+ ResultScanner rs = tt.getScanner(t2, new Scan());
+ Result r = rs.next();
+ count = 0;
+ while (r != null) {
+ count++;
+ LOG.trace("row: " + Bytes.toString(r.getRow()) + " count: " + count);
+ r = rs.next();
+ }
+ assertEquals("Wrong count", rowcount - 1, count);
+
+ Transaction t3 = tm.begin();
+ LOG.info("Transaction created " + t3);
+ Put p = new Put(modrow);
+ p.add(fam, col, data2);
+ tt.put(t3, p);
+
+ tm.commit(t3);
+
+ boolean aborted = false;
+ try {
+ tm.commit(t2);
+ assertTrue("Didn't abort", false);
+ } catch (RollbackException e) {
+ aborted = true;
+ }
+ assertTrue("Didn't raise exception", aborted);
+
+ Transaction tscan = tm.begin();
+ rs = tt.getScanner(tscan, new Scan());
+ r = rs.next();
+ count = 0;
+ while (r != null) {
+ count++;
+ r = rs.next();
+ }
+ assertEquals("Wrong count", rowcount, count);
+
+ } catch (Exception e) {
+ LOG.error("Exception occurred", e);
+ throw e;
+ }
+ }
+
+ @Test
+ public void testMultipleCellChangesOnSameRow(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ Transaction t2 = tm.begin();
+ LOG.info("Transactions created " + t1 + " " + t2);
+
+ byte[] row = Bytes.toBytes("row");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col1 = Bytes.toBytes("testdata1");
+ byte[] col2 = Bytes.toBytes("testdata2");
+ byte[] data = Bytes.toBytes("testWrite-1");
+
+ Put p2 = new Put(row);
+ p2.add(fam, col1, data);
+ tt.put(t2, p2);
+ tm.commit(t2);
+
+ Put p1 = new Put(row);
+ p1.add(fam, col2, data);
+ tt.put(t1, p1);
+ tm.commit(t1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestTxMgrFailover.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestTxMgrFailover.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestTxMgrFailover.java
new file mode 100644
index 0000000..13428cd
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestTxMgrFailover.java
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.TestUtils;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.committable.InMemoryCommitTable;
+import org.apache.omid.transaction.Transaction.Status;
+import org.apache.omid.tso.ProgrammableTSOServer;
+import org.apache.omid.tso.client.TSOClient;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.COMMIT_TABLE;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+@Test(groups = "sharedHBase")
+public class TestTxMgrFailover extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTxMgrFailover.class);
+
+ private static final int TSO_SERVER_PORT = 3333;
+ private static final String TSO_SERVER_HOST = "localhost";
+
+ private static final long TX1_ST = 1L;
+ private static final long TX1_CT = 2L;
+
+ private static final byte[] qualifier = Bytes.toBytes("test-qual");
+ private static final byte[] row1 = Bytes.toBytes("row1");
+ private static final byte[] data1 = Bytes.toBytes("testWrite-1");
+
+ // Used in test assertions
+ private InMemoryCommitTable commitTable;
+
+ private CommitTable.Client commitTableClient;
+ private CommitTable.Writer commitTableWriter;
+
+ // Allows to prepare the required responses to client request operations
+ private ProgrammableTSOServer tso;
+
+ // The transaction manager under test
+ private HBaseTransactionManager tm;
+
+ @BeforeClass(alwaysRun = true)
+ public void beforeClass() throws Exception {
+ // ------------------------------------------------------------------------------------------------------------
+ // ProgrammableTSOServer setup
+ // ------------------------------------------------------------------------------------------------------------
+ tso = new ProgrammableTSOServer(TSO_SERVER_PORT);
+ TestUtils.waitForSocketListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 100);
+ }
+
+ @BeforeMethod(alwaysRun = true, timeOut = 30_000)
+ public void beforeMethod() throws IOException, InterruptedException {
+
+ commitTable = new InMemoryCommitTable(); // Use an in-memory commit table to speed up tests
+ commitTableClient = spy(commitTable.getClient());
+ commitTableWriter = spy(commitTable.getWriter());
+
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ TSOClient tsoClientForTM = spy(TSOClient.newInstance(hbaseOmidClientConf.getOmidClientConfiguration()));
+
+ tm = spy(HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .tsoClient(tsoClientForTM)
+ .commitTableClient(commitTableClient)
+ .build());
+ }
+
+ @Test(timeOut = 30_000)
+ public void testAbortResponseFromTSOThrowsRollbackExceptionInClient() throws Exception {
+ // Program the TSO to return an ad-hoc Timestamp and an abort response for tx 1
+ tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
+ tso.queueResponse(new ProgrammableTSOServer.AbortResponse(TX1_ST));
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ assertEquals(tx1.getStartTimestamp(), TX1_ST);
+ Put put = new Put(row1);
+ put.add(TEST_FAMILY.getBytes(), qualifier, data1);
+ txTable.put(tx1, put);
+ assertEquals(hBaseUtils.countRows(new HTable(hbaseConf, TEST_TABLE)), 1, "Rows should be 1!");
+ checkOperationSuccessOnCell(KeyValue.Type.Put, data1, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
+ qualifier);
+
+ try {
+ tm.commit(tx1);
+ fail();
+ } catch (RollbackException e) {
+ // Expected!
+
+ }
+
+ // Check transaction status
+ assertEquals(tx1.getStatus(), Status.ROLLEDBACK);
+ assertEquals(tx1.getCommitTimestamp(), 0);
+ // Check the cleanup process did its job and the committed data is NOT there
+ checkOperationSuccessOnCell(KeyValue.Type.Delete, null, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
+ qualifier);
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testClientReceivesSuccessfulCommitForNonInvalidatedTxCommittedByPreviousTSO() throws Exception {
+
+ // Program the TSO to return an ad-hoc Timestamp and an commit response with heuristic actions
+ tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
+ tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
+ // Simulate that tx1 was committed by writing to commit table
+ commitTableWriter.addCommittedTransaction(TX1_ST, TX1_CT);
+ commitTableWriter.flush();
+ assertEquals(commitTable.countElements(), 1, "Rows should be 1!");
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ assertEquals(tx1.getStartTimestamp(), TX1_ST);
+ Put put = new Put(row1);
+ put.add(TEST_FAMILY.getBytes(), qualifier, data1);
+ txTable.put(tx1, put);
+ // Should succeed
+ tm.commit(tx1);
+
+ // Check transaction status
+ assertEquals(tx1.getStatus(), Status.COMMITTED);
+ assertEquals(tx1.getCommitTimestamp(), TX1_CT);
+ // Check the cleanup process did its job and the committed data is there
+ // Note that now we do not clean up the commit table when exercising the heuristic actions
+ assertEquals(commitTable.countElements(), 1,
+ "Rows should be 1! We don't have to clean CT in this case");
+ Optional<CommitTimestamp>
+ optionalCT =
+ tm.commitTableClient.getCommitTimestamp(TX1_ST).get();
+ assertTrue(optionalCT.isPresent());
+ checkOperationSuccessOnCell(KeyValue.Type.Put, data1, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
+ qualifier);
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testClientReceivesRollbackExceptionForInvalidatedTxCommittedByPreviousTSO() throws Exception {
+
+ // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
+ tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
+ tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
+ // Simulate that tx1 was committed by writing to commit table but was later invalidated
+ commitTableClient.tryInvalidateTransaction(TX1_ST);
+ assertEquals(commitTable.countElements(), 1, "Rows should be 1!");
+
+ executeTxAndCheckRollback();
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testClientReceivesNotificationOfANewTSOCanInvalidateTransaction() throws Exception {
+
+ // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
+ tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
+ tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
+
+ assertEquals(commitTable.countElements(), 0, "Rows should be 0!");
+
+ executeTxAndCheckRollback();
+
+ }
+
+ private void executeTxAndCheckRollback() throws IOException, TransactionException, InterruptedException, java.util.concurrent.ExecutionException {
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ assertEquals(tx1.getStartTimestamp(), TX1_ST);
+ Put put = new Put(row1);
+ put.add(TEST_FAMILY.getBytes(), qualifier, data1);
+ txTable.put(tx1, put);
+ try {
+ tm.commit(tx1);
+ fail();
+ } catch (RollbackException e) {
+ // Expected
+ }
+
+ // Check transaction status
+ assertEquals(tx1.getStatus(), Status.ROLLEDBACK);
+ assertEquals(tx1.getCommitTimestamp(), 0);
+ // Check the cleanup process did its job and the uncommitted data is NOT there
+ assertEquals(commitTable.countElements(), 1, "Rows should be 1! Dirty data should be there");
+ Optional<CommitTimestamp>
+ optionalCT =
+ tm.commitTableClient.getCommitTimestamp(TX1_ST).get();
+ assertTrue(optionalCT.isPresent());
+ assertFalse(optionalCT.get().isValid());
+ checkOperationSuccessOnCell(KeyValue.Type.Delete, null, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
+ qualifier);
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void testClientSuccessfullyCommitsWhenReceivingNotificationOfANewTSOAandCANTInvalidateTransaction()
+ throws Exception {
+
+ // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
+ tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
+ tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
+
+ // Simulate that the original TSO was able to add the tx to commit table in the meantime
+ commitTableWriter.addCommittedTransaction(TX1_ST, TX1_CT);
+ commitTableWriter.flush();
+ assertEquals(commitTable.countElements(), 1, "Rows should be 1!");
+ SettableFuture<Optional<CommitTimestamp>> f1 = SettableFuture.create();
+ f1.set(Optional.<CommitTimestamp>absent());
+ SettableFuture<Optional<CommitTimestamp>> f2 = SettableFuture.create();
+ f2.set(Optional.of(new CommitTimestamp(COMMIT_TABLE, TX1_CT, true)));
+ doReturn(f1).doReturn(f2).when(commitTableClient).getCommitTimestamp(TX1_ST);
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ assertEquals(tx1.getStartTimestamp(), TX1_ST);
+ Put put = new Put(row1);
+ put.add(TEST_FAMILY.getBytes(), qualifier, data1);
+ txTable.put(tx1, put);
+
+ tm.commit(tx1);
+
+ // Check transaction status
+ assertEquals(tx1.getStatus(), Status.COMMITTED);
+ assertEquals(tx1.getCommitTimestamp(), TX1_CT);
+ // Check the cleanup process did its job and the committed data is there
+ // Note that now we do not clean up the commit table when exercising the heuristic actions
+ assertEquals(commitTable.countElements(), 1,
+ "Rows should be 1! We don't have to clean CT in this case");
+ checkOperationSuccessOnCell(KeyValue.Type.Put, data1, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
+ qualifier);
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testClientReceivesATransactionExceptionWhenReceivingNotificationOfANewTSOAndCANTInvalidateTransactionAndCTCheckIsUnsuccessful()
+ throws Exception {
+
+ // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
+ tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
+ tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
+
+ // Simulate that the original TSO was able to add the tx to commit table in the meantime
+ SettableFuture<Boolean> f = SettableFuture.create();
+ f.set(false);
+ doReturn(f).when(commitTableClient).tryInvalidateTransaction(TX1_ST);
+
+ assertEquals(commitTable.countElements(), 0, "Rows should be 0!");
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ assertEquals(tx1.getStartTimestamp(), TX1_ST);
+ Put put = new Put(row1);
+ put.add(TEST_FAMILY.getBytes(), qualifier, data1);
+ txTable.put(tx1, put);
+ try {
+ tm.commit(tx1);
+ fail();
+ } catch (TransactionException e) {
+ // Expected but is not good because we're not able to determine the tx outcome
+ }
+
+ // Check transaction status
+ assertEquals(tx1.getStatus(), Status.RUNNING);
+ assertEquals(tx1.getCommitTimestamp(), 0);
+ }
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ protected void checkOperationSuccessOnCell(KeyValue.Type targetOp,
+ @Nullable byte[] expectedValue,
+ byte[] tableName,
+ byte[] row,
+ byte[] fam,
+ byte[] col) {
+
+ try (HTable table = new HTable(hbaseConf, tableName)) {
+ Get get = new Get(row).setMaxVersions(1);
+ Result result = table.get(get);
+ Cell latestCell = result.getColumnLatestCell(fam, col);
+
+ switch (targetOp) {
+ case Put:
+ assertEquals(latestCell.getTypeByte(), targetOp.getCode());
+ assertEquals(CellUtil.cloneValue(latestCell), expectedValue);
+ LOG.trace("Value for " + Bytes.toString(tableName) + ":"
+ + Bytes.toString(row) + ":" + Bytes.toString(fam) + ":"
+ + Bytes.toString(col) + "=>" + Bytes.toString(CellUtil.cloneValue(latestCell))
+ + " (" + Bytes.toString(expectedValue) + " expected)");
+ break;
+ case Delete:
+ LOG.trace("Value for " + Bytes.toString(tableName) + ":"
+ + Bytes.toString(row) + ":" + Bytes.toString(fam)
+ + Bytes.toString(col) + " deleted");
+ assertNull(latestCell);
+ break;
+ default:
+ fail();
+ }
+ } catch (IOException e) {
+ LOG.error("Error reading row " + Bytes.toString(tableName) + ":"
+ + Bytes.toString(row) + ":" + Bytes.toString(fam)
+ + Bytes.toString(col), e);
+ fail();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestUpdateScan.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestUpdateScan.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestUpdateScan.java
new file mode 100644
index 0000000..342f349
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestUpdateScan.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.WhileMatchFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestUpdateScan extends OmidTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(TestUpdateScan.class);
+
+ private static final String TEST_COL = "value";
+ private static final String TEST_COL_2 = "col_2";
+
+ @Test
+ public void testGet(ITestContext context) throws Exception {
+ try {
+ TransactionManager tm = newTransactionManager(context);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+ Transaction t = tm.begin();
+ int[] lInts = new int[]{100, 243, 2342, 22, 1, 5, 43, 56};
+ for (int i = 0; i < lInts.length; i++) {
+ byte[] data = Bytes.toBytes(lInts[i]);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
+ table.put(t, put);
+ }
+ int startKeyValue = lInts[3];
+ int stopKeyValue = lInts[3];
+ byte[] startKey = Bytes.toBytes(startKeyValue);
+ byte[] stopKey = Bytes.toBytes(stopKeyValue);
+ Get g = new Get(startKey);
+ Result r = table.get(t, g);
+ if (!r.isEmpty()) {
+ int tmp = Bytes.toInt(r.getValue(Bytes.toBytes(TEST_FAMILY),
+ Bytes.toBytes(TEST_COL)));
+ LOG.info("Result:" + tmp);
+ assertTrue("Bad value, should be "
+ + startKeyValue + " but is " + tmp
+ , tmp == startKeyValue);
+ } else {
+ Assert.fail("Bad result");
+ }
+ tm.commit(t);
+
+ Scan s = new Scan(startKey);
+ CompareFilter.CompareOp op = CompareFilter.CompareOp.LESS_OR_EQUAL;
+ RowFilter toFilter = new RowFilter(op, new BinaryPrefixComparator(stopKey));
+ boolean startInclusive = true;
+ if (!startInclusive) {
+ FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
+ filters.addFilter(new RowFilter(CompareFilter.CompareOp.GREATER,
+ new BinaryPrefixComparator(startKey)));
+ filters.addFilter(new WhileMatchFilter(toFilter));
+ s.setFilter(filters);
+ } else {
+ s.setFilter(new WhileMatchFilter(toFilter));
+ }
+ t = tm.begin();
+ ResultScanner res = table.getScanner(t, s);
+ Result rr;
+ int count = 0;
+ while ((rr = res.next()) != null) {
+ int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
+ Bytes.toBytes(TEST_COL)));
+ LOG.info("Result: " + iTmp);
+ count++;
+ }
+ assertEquals("Count is wrong", 1, count);
+ LOG.info("Rows found " + count);
+ tm.commit(t);
+ table.close();
+ } catch (Exception e) {
+ LOG.error("Exception in test", e);
+ }
+ }
+
+ @Test
+ public void testScan(ITestContext context) throws Exception {
+
+ try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
+ TransactionManager tm = newTransactionManager(context);
+ Transaction t = tm.begin();
+ int[] lInts = new int[]{100, 243, 2342, 22, 1, 5, 43, 56};
+ for (int lInt : lInts) {
+ byte[] data = Bytes.toBytes(lInt);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL_2), data);
+ table.put(t, put);
+ }
+
+ Scan s = new Scan();
+ // Adding two columns to the scanner should not throw a
+ // ConcurrentModificationException when getting the scanner
+ s.addColumn(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL));
+ s.addColumn(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL_2));
+ ResultScanner res = table.getScanner(t, s);
+ Result rr;
+ int count = 0;
+ while ((rr = res.next()) != null) {
+ int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
+ Bytes.toBytes(TEST_COL)));
+ LOG.info("Result: " + iTmp);
+ count++;
+ }
+ assertTrue("Count should be " + lInts.length + " but is " + count,
+ count == lInts.length);
+ LOG.info("Rows found " + count);
+
+ tm.commit(t);
+
+ t = tm.begin();
+ res = table.getScanner(t, s);
+ count = 0;
+ while ((rr = res.next()) != null) {
+ int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
+ Bytes.toBytes(TEST_COL)));
+ LOG.info("Result: " + iTmp);
+ count++;
+ }
+ assertTrue("Count should be " + lInts.length + " but is " + count,
+ count == lInts.length);
+ LOG.info("Rows found " + count);
+ tm.commit(t);
+ }
+
+ }
+
+
+ @Test
+ public void testScanUncommitted(ITestContext context) throws Exception {
+ try {
+ TransactionManager tm = newTransactionManager(context);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+ Transaction t = tm.begin();
+ int[] lIntsA = new int[]{100, 243, 2342, 22, 1, 5, 43, 56};
+ for (int aLIntsA : lIntsA) {
+ byte[] data = Bytes.toBytes(aLIntsA);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
+ table.put(t, put);
+ }
+ tm.commit(t);
+
+ Transaction tu = tm.begin();
+ int[] lIntsB = new int[]{105, 24, 4342, 32, 7, 3, 30, 40};
+ for (int aLIntsB : lIntsB) {
+ byte[] data = Bytes.toBytes(aLIntsB);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
+ table.put(tu, put);
+ }
+
+ t = tm.begin();
+ int[] lIntsC = new int[]{109, 224, 242, 2, 16, 59, 23, 26};
+ for (int aLIntsC : lIntsC) {
+ byte[] data = Bytes.toBytes(aLIntsC);
+ Put put = new Put(data);
+ put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
+ table.put(t, put);
+ }
+ tm.commit(t);
+
+ t = tm.begin();
+ Scan s = new Scan();
+ ResultScanner res = table.getScanner(t, s);
+ Result rr;
+ int count = 0;
+
+ while ((rr = res.next()) != null) {
+ int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
+ Bytes.toBytes(TEST_COL)));
+ LOG.info("Result: " + iTmp);
+ count++;
+ }
+ assertTrue("Count should be " + (lIntsA.length * lIntsC.length) + " but is " + count,
+ count == lIntsA.length + lIntsC.length);
+ LOG.info("Rows found " + count);
+ tm.commit(t);
+ table.close();
+ } catch (Exception e) {
+ LOG.error("Exception in test", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/resources/log4j.properties b/hbase-client/src/test/resources/log4j.properties
index 90954b7..6380e8a 100644
--- a/hbase-client/src/test/resources/log4j.properties
+++ b/hbase-client/src/test/resources/log4j.properties
@@ -36,13 +36,12 @@ log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss,SSS} [%t] %
log4j.logger.org.apache.zookeeper=INFO
log4j.logger.org.apache.hadoop=INFO
log4j.logger.org.apache.hadoop.ipc=ERROR
-
-log4j.logger.com.yahoo.omid=INFO
-#log4j.logger.com.yahoo.omid.regionserver.TransactionalRegionServer=TRACE
-#log4j.logger.com.yahoo.omid.TestBasicTransaction=TRACE
-#log4j.logger.com.yahoo.omid.client.TSOClient=TRACE
-#log4j.logger.com.yahoo.omid.client.TransactionState=TRACE
-#log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=TRACE
+log4j.logger.org.apache.omid=INFO
+#log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE
+#log4j.logger.org.apache.omid.TestBasicTransaction=TRACE
+#log4j.logger.org.apache.omid.client.TSOClient=TRACE
+#log4j.logger.org.apache.omid.client.TransactionState=TRACE
+#log4j.logger.org.apache.omid.tso.ThroughputMonitor=TRACE
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/resources/test-hbase-omid-client-config.yml b/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
index 9a95df3..8b434da 100644
--- a/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
+++ b/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
@@ -2,10 +2,10 @@
commitTableName: OMID_COMMIT_TABLE1
#TSO/HA connection
-omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
+omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration
#TSO/HA connection
connectionString: "somehost:54758"
- connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType HA
+ connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType HA
zkConnectionTimeoutInSecs: 11
#TSO related
@@ -16,18 +16,18 @@ omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
executorThreads: 4
#Instrumentation
-metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+ !!org.apache.omid.metrics.CodahaleMetricsConfig {
outputFreqInSecs: 1,
reporters: !!set {
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter SLF4J,
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter GRAPHITE,
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter SLF4J,
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter GRAPHITE,
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
},
csvDir: "some/folder",
prefix: "somePrefix",
- slf4jLogger: "com.yahoo",
+ slf4jLogger: "org.apache",
graphiteHostConfig: "somehost:1234"
}
]
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index 0ddfafd..82dc0f7 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -17,17 +17,17 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-common</artifactId>
<version>${project.version}</version>
</dependency>
@@ -129,7 +129,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -140,7 +140,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
[38/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
deleted file mode 100644
index 3e39d69..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-import com.yahoo.omid.HBaseShims;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedMap;
-
-import static com.yahoo.omid.transaction.CellUtils.SHADOW_CELL_SUFFIX;
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertTrue;
-import static org.testng.AssertJUnit.fail;
-
-@Test(groups = "noHBase")
-public class TestCellUtils {
-
- private final byte[] row = Bytes.toBytes("test-row");
- private final byte[] family = Bytes.toBytes("test-family");
- private final byte[] qualifier = Bytes.toBytes("test-qual");
- private final byte[] otherQualifier = Bytes.toBytes("other-test-qual");
-
- @DataProvider(name = "shadow-cell-suffixes")
- public Object[][] createShadowCellSuffixes() {
- return new Object[][]{
- {SHADOW_CELL_SUFFIX},
- };
- }
-
- @Test(dataProvider = "shadow-cell-suffixes")
- public void testShadowCellQualifiers(byte[] shadowCellSuffixToTest) throws IOException {
-
- final byte[] validShadowCellQualifier =
- com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
- final byte[] sandwichValidShadowCellQualifier =
- com.google.common.primitives.Bytes.concat(shadowCellSuffixToTest, validShadowCellQualifier);
- final byte[] doubleEndedValidShadowCellQualifier =
- com.google.common.primitives.Bytes.concat(validShadowCellQualifier, shadowCellSuffixToTest);
- final byte[] interleavedValidShadowCellQualifier =
- com.google.common.primitives.Bytes.concat(validShadowCellQualifier,
- com.google.common.primitives.Bytes
- .concat(validShadowCellQualifier, validShadowCellQualifier));
- final byte[] value = Bytes.toBytes("test-value");
-
- // Test the qualifier passed is a shadow cell
- // qualifier because it contains only one suffix
- // and is placed at the end of the qualifier:
- // qual_nameSUFFIX
- KeyValue kv = new KeyValue(row, family, validShadowCellQualifier, value);
- assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
-
- // We also accept this pattern in the qualifier:
- // SUFFIXqual_nameSUFFIX
- kv = new KeyValue(row, family, sandwichValidShadowCellQualifier, value);
- assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
-
- // We also accept this pattern in the qualifier:
- // qual_nameSUFFIXSUFFIX
- kv = new KeyValue(row, family, doubleEndedValidShadowCellQualifier, value);
- assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
-
- // We also accept this pattern in the qualifier:
- // qual_nameSUFFIXqual_nameSUFFIXqual_nameSUFFIX
- kv = new KeyValue(row, family, interleavedValidShadowCellQualifier, value);
- assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
-
- // Test the qualifier passed is not a shadow cell
- // qualifier if there's nothing else apart from the suffix
- kv = new KeyValue(row, family, shadowCellSuffixToTest, value);
- assertFalse("Should not include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
-
- }
-
- @Test
- public void testCorrectMapingOfCellsToShadowCells() throws IOException {
- // Create the required data
- final byte[] validShadowCellQualifier =
- com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
-
- final byte[] qualifier2 = Bytes.toBytes("test-qual2");
- final byte[] validShadowCellQualifier2 =
- com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX);
-
- final byte[] qualifier3 = Bytes.toBytes("test-qual3");
-
- Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
- Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
- Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
- Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
- Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
-
- Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
- Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
-
- Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
-
- // Check a list of cells with duplicate values
- List<Cell> badListWithDups = new ArrayList<>();
- badListWithDups.add(cell1);
- badListWithDups.add(dupCell1WithAnotherValue);
-
- // Check dup shadow cell with same MVCC is ignored
- SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
- assertEquals("There should be only 1 key-value maps", 1, cellsToShadowCells.size());
- assertTrue(cellsToShadowCells.containsKey(cell1));
- KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
- KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
- assertTrue(firstKey.equals(lastKey));
- assertTrue("Should be equal", 0 == Bytes.compareTo(
- firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
- cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()));
-
- // Modify dup shadow cell to have a greater MVCC and check that is replaced
- HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
- cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
- assertEquals("There should be only 1 key-value maps", 1, cellsToShadowCells.size());
- assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
- firstKey = (KeyValue) cellsToShadowCells.firstKey();
- lastKey = (KeyValue) cellsToShadowCells.lastKey();
- assertTrue(firstKey.equals(lastKey));
- assertTrue("Should be equal", 0 == Bytes.compareTo(
- firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
- dupCell1WithAnotherValue.getValueArray(), dupCell1WithAnotherValue.getValueOffset(),
- dupCell1WithAnotherValue.getValueLength()));
- // Check a list of cells with duplicate values
- List<Cell> cellListWithDups = new ArrayList<>();
- cellListWithDups.add(cell1);
- cellListWithDups.add(shadowCell1);
- cellListWithDups.add(dupCell1); // Dup cell
- cellListWithDups.add(delCell1); // Another Dup cell but with different type
- cellListWithDups.add(cell2);
- cellListWithDups.add(cell3);
- cellListWithDups.add(shadowCell2);
-
- cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
- assertEquals("There should be only 3 key-value maps", 3, cellsToShadowCells.size());
- assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
- assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
- assertFalse(cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
- // The current algo avoids to put the delete cell
- // as key after the put cell with same value was added
- assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
- assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
-
- }
-
- @Test
- public void testShadowCellSuffixConcatenationToQualifier() {
-
- Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
- byte[] suffixedQualifier = CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
- assertEquals(expectedQualifier, suffixedQualifier);
-
- }
-
- @Test(dataProvider = "shadow-cell-suffixes")
- public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException {
-
- // Test removal from a correclty suffixed qualifier
- byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
- Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
- byte[] resultedQualifier = CellUtils.removeShadowCellSuffix(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- byte[] expectedQualifier = qualifier;
- assertEquals(expectedQualifier, resultedQualifier);
-
- // Test removal from a badly suffixed qualifier
- byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, Bytes.toBytes("BAD"));
- Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
- try {
- CellUtils.removeShadowCellSuffix(badCell.getQualifierArray(),
- badCell.getQualifierOffset(),
- badCell.getQualifierLength());
- fail();
- } catch (IllegalArgumentException e) {
- // Expected
- }
- }
-
- @Test
- public void testMatchingQualifiers() {
- Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
- assertTrue(CellUtils.matchingQualifier(cell, qualifier, 0, qualifier.length));
- assertFalse(CellUtils.matchingQualifier(cell, otherQualifier, 0, otherQualifier.length));
- }
-
- @Test(dataProvider = "shadow-cell-suffixes")
- public void testQualifierLengthFromShadowCellQualifier(byte[] shadowCellSuffixToTest) {
- // Test suffixed qualifier
- byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
- int originalQualifierLength =
- CellUtils.qualifierLengthFromShadowCellQualifier(suffixedQualifier, 0, suffixedQualifier.length);
- assertEquals(qualifier.length, originalQualifierLength);
-
- // Test passing qualifier without shadow cell suffix
- originalQualifierLength =
- CellUtils.qualifierLengthFromShadowCellQualifier(qualifier, 0, qualifier.length);
- assertEquals(qualifier.length, originalQualifierLength);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
deleted file mode 100644
index e57c7fe..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.mortbay.log.Log;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.testng.AssertJUnit.assertEquals;
-
-@Test(groups = "noHBase")
-public class TestColumnIterator {
-
- final byte[] row = Bytes.toBytes("row");
- private final byte[] family1 = Bytes.toBytes("f1");
- private final byte[] family2 = Bytes.toBytes("f2");
- private final byte[] qualifier1 = Bytes.toBytes("c1");
- private final byte[] qualifier2 = Bytes.toBytes("c2");
- final byte[] data = Bytes.toBytes("data");
-
- private final List<Cell> cells = new ArrayList<Cell>(
- Arrays.asList(
- // Group 1 (3 elems but grouping should filter shadow cell, so check for 2)
- new KeyValue(row, family1, qualifier1, 0, data),
- new KeyValue(row, family1, qualifier1, 1, data),
- new KeyValue(row, family1, CellUtils.addShadowCellSuffix(qualifier1), 0, data),
- // Group 2 (2 elems but grouping should filter shadow cell, so check for 1)
- new KeyValue(row, family1, qualifier2, 0, data),
- new KeyValue(row, family1, CellUtils.addShadowCellSuffix(qualifier2), 0, data),
- // Group 3 (2 elems but grouping should filter shadow cell, so check for 1)
- new KeyValue(row, family2, qualifier1, 0, data),
- new KeyValue(row, family2, CellUtils.addShadowCellSuffix(qualifier1), 0, data)
- )
- );
-
- @Test
- public void testGroupingCellsByColumnFilteringShadowCells() {
-
- ImmutableList<Collection<Cell>> groupedColumnsWithoutShadowCells =
- TTable.groupCellsByColumnFilteringShadowCells(cells);
- Log.info("Column Groups " + groupedColumnsWithoutShadowCells);
- assertEquals("Should be 3 column groups", 3, groupedColumnsWithoutShadowCells.size());
- int group1Counter = 0;
- int group2Counter = 0;
- int group3Counter = 0;
- for (Collection<Cell> columns : groupedColumnsWithoutShadowCells) {
- for (Cell cell : columns) {
- byte[] cellFamily = CellUtil.cloneFamily(cell);
- byte[] cellQualifier = CellUtil.cloneQualifier(cell);
- // Group 1
- if (Bytes.equals(cellFamily, family1) &&
- Bytes.equals(cellQualifier, qualifier1)) {
- group1Counter++;
- }
- // Group 2
- if (Bytes.equals(cellFamily, family1) &&
- Bytes.equals(cellQualifier, qualifier2)) {
- group2Counter++;
- }
- // Group 3
- if (Bytes.equals(cellFamily, family2) &&
- Bytes.equals(cellQualifier, qualifier1)) {
- group3Counter++;
- }
- }
- }
-
- assertEquals("Group 1 should have 2 elems", 2, group1Counter);
- assertEquals("Group 2 should have 1 elems", 1, group2Counter);
- assertEquals("Group 3 should have 1 elems", 1, group3Counter);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
deleted file mode 100644
index c3c24bb..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
+++ /dev/null
@@ -1,299 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.AssertJUnit;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.testng.Assert.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestDeletion extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestDeletion.class);
-
- private byte[] famA = Bytes.toBytes(TEST_FAMILY);
- private byte[] famB = Bytes.toBytes(TEST_FAMILY2);
- private byte[] colA = Bytes.toBytes("testdataA");
- private byte[] colB = Bytes.toBytes("testdataB");
- private byte[] data1 = Bytes.toBytes("testWrite-1");
- private byte[] modrow = Bytes.toBytes("test-del" + 3);
-
- private static class FamCol {
-
- final byte[] fam;
- final byte[] col;
-
- FamCol(byte[] fam, byte[] col) {
- this.fam = fam;
- this.col = col;
- }
-
- }
-
- @Test
- public void runTestDeleteFamily(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- int rowsWritten = 10;
- FamCol famColA = new FamCol(famA, colA);
- FamCol famColB = new FamCol(famB, colB);
- writeRows(tt, t1, rowsWritten, famColA, famColB);
- tm.commit(t1);
-
- Transaction t2 = tm.begin();
- Delete d = new Delete(modrow);
- d.deleteFamily(famA);
- tt.delete(t2, d);
-
- Transaction tscan = tm.begin();
- ResultScanner rs = tt.getScanner(tscan, new Scan());
-
- Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
- AssertJUnit.assertEquals("ColA count should be equal to rowsWritten", rowsWritten, (int) count.get(famColA));
- AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
- tm.commit(t2);
-
- tscan = tm.begin();
- rs = tt.getScanner(tscan, new Scan());
-
- count = countColsInRows(rs, famColA, famColB);
- AssertJUnit
- .assertEquals("ColA count should be equal to rowsWritten - 1", (rowsWritten - 1), (int) count.get(famColA));
- AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
- }
-
- @Test
- public void runTestDeleteColumn(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- int rowsWritten = 10;
-
- FamCol famColA = new FamCol(famA, colA);
- FamCol famColB = new FamCol(famA, colB);
- writeRows(tt, t1, rowsWritten, famColA, famColB);
- tm.commit(t1);
-
- Transaction t2 = tm.begin();
- Delete d = new Delete(modrow);
- d.deleteColumn(famA, colA);
- tt.delete(t2, d);
-
- Transaction tscan = tm.begin();
- ResultScanner rs = tt.getScanner(tscan, new Scan());
-
- Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
- AssertJUnit.assertEquals("ColA count should be equal to rowsWritten", rowsWritten, (int) count.get(famColA));
- AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
- tm.commit(t2);
-
- tscan = tm.begin();
- rs = tt.getScanner(tscan, new Scan());
-
- count = countColsInRows(rs, famColA, famColB);
- AssertJUnit
- .assertEquals("ColA count should be equal to rowsWritten - 1", (rowsWritten - 1), (int) count.get(famColA));
- AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
- }
-
- /**
- * This test is very similar to #runTestDeleteColumn() but exercises Delete#deleteColumns()
- */
- @Test
- public void runTestDeleteColumns(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- int rowsWritten = 10;
-
- FamCol famColA = new FamCol(famA, colA);
- FamCol famColB = new FamCol(famA, colB);
- writeRows(tt, t1, rowsWritten, famColA, famColB);
- tm.commit(t1);
-
- Transaction t2 = tm.begin();
- Delete d = new Delete(modrow);
- d.deleteColumns(famA, colA);
- tt.delete(t2, d);
-
- Transaction tscan = tm.begin();
- ResultScanner rs = tt.getScanner(tscan, new Scan());
-
- Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
- AssertJUnit.assertEquals("ColA count should be equal to rowsWritten", rowsWritten, (int) count.get(famColA));
- AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
- tm.commit(t2);
-
- tscan = tm.begin();
- rs = tt.getScanner(tscan, new Scan());
-
- count = countColsInRows(rs, famColA, famColB);
-
- AssertJUnit
- .assertEquals("ColA count should be equal to rowsWritten - 1", (rowsWritten - 1), (int) count.get(famColA));
- AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
- }
-
- @Test
- public void runTestDeleteRow(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- int rowsWritten = 10;
-
- FamCol famColA = new FamCol(famA, colA);
- writeRows(tt, t1, rowsWritten, famColA);
-
- tm.commit(t1);
-
- Transaction t2 = tm.begin();
- Delete d = new Delete(modrow);
- tt.delete(t2, d);
-
- Transaction tscan = tm.begin();
- ResultScanner rs = tt.getScanner(tscan, new Scan());
-
- int rowsRead = countRows(rs);
- AssertJUnit.assertTrue("Expected " + rowsWritten + " rows but " + rowsRead + " found",
- rowsRead == rowsWritten);
-
- tm.commit(t2);
-
- tscan = tm.begin();
- rs = tt.getScanner(tscan, new Scan());
-
- rowsRead = countRows(rs);
- AssertJUnit.assertTrue("Expected " + (rowsWritten - 1) + " rows but " + rowsRead + " found",
- rowsRead == (rowsWritten - 1));
-
- }
-
- @Test
- public void testDeletionOfNonExistingColumnFamilyDoesNotWriteToHBase(ITestContext context) throws Exception {
-
- // --------------------------------------------------------------------
- // Setup initial environment for the test
- // --------------------------------------------------------------------
- TransactionManager tm = newTransactionManager(context);
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction tx1 = tm.begin();
- LOG.info("{} writing initial data created ", tx1);
- Put p = new Put(Bytes.toBytes("row1"));
- p.add(famA, colA, data1);
- txTable.put(tx1, p);
- tm.commit(tx1);
-
- // --------------------------------------------------------------------
- // Try to delete a non existing CF
- // --------------------------------------------------------------------
- Transaction deleteTx = tm.begin();
- LOG.info("{} trying to delete a non-existing family created ", deleteTx);
- Delete del = new Delete(Bytes.toBytes("row1"));
- del.deleteFamily(famB);
- // This delete should not put data on HBase
- txTable.delete(deleteTx, del);
-
- // --------------------------------------------------------------------
- // Check data has not been written to HBase
- // --------------------------------------------------------------------
- HTable table = new HTable(hbaseConf, TEST_TABLE);
- Get get = new Get(Bytes.toBytes("row1"));
- get.setTimeStamp(deleteTx.getTransactionId());
- Result result = table.get(get);
- assertTrue(result.isEmpty());
-
- }
-
- private int countRows(ResultScanner rs) throws IOException {
- int count;
- Result r = rs.next();
- count = 0;
- while (r != null) {
- count++;
- LOG.trace("row: " + Bytes.toString(r.getRow()) + " count: " + count);
- r = rs.next();
- }
- return count;
- }
-
- private void writeRows(TTable tt, Transaction t1, int rowcount, FamCol... famCols) throws IOException {
- for (int i = 0; i < rowcount; i++) {
- byte[] row = Bytes.toBytes("test-del" + i);
-
- Put p = new Put(row);
- for (FamCol col : famCols) {
- p.add(col.fam, col.col, data1);
- }
- tt.put(t1, p);
- }
- }
-
- private Map<FamCol, Integer> countColsInRows(ResultScanner rs, FamCol... famCols) throws IOException {
- Map<FamCol, Integer> colCount = new HashMap<>();
- Result r = rs.next();
- while (r != null) {
- for (FamCol col : famCols) {
- if (r.containsColumn(col.fam, col.col)) {
- Integer c = colCount.get(col);
-
- if (c == null) {
- colCount.put(col, 1);
- } else {
- colCount.put(col, c + 1);
- }
- }
- }
- r = rs.next();
- }
- return colCount;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
deleted file mode 100644
index 0434520..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
+++ /dev/null
@@ -1,412 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Charsets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.tso.LeaseManagement;
-import com.yahoo.omid.tso.PausableLeaseManager;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.framework.recipes.cache.NodeCache;
-import org.apache.curator.framework.recipes.cache.NodeCacheListener;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
-import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
-import static com.yahoo.omid.tso.client.OmidClientConfiguration.ConnType.HA;
-import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-@Test(groups = "sharedHBase")
-public class TestEndToEndScenariosWithHA extends OmidTestBase {
-
- private static final int TEST_LEASE_PERIOD_MS = 5_000;
- private static final String CURRENT_TSO_PATH = "/CURRENT_TSO_PATH";
- private static final String TSO_LEASE_PATH = "/TSO_LEASE_PATH";
- private static final String NAMESPACE = "omid";
-
- private static final Logger LOG = LoggerFactory.getLogger(TestEndToEndScenariosWithHA.class);
-
- private static final byte[] qualifier1 = Bytes.toBytes("test-q1");
- private static final byte[] qualifier2 = Bytes.toBytes("test-q2l");
- private static final byte[] row1 = Bytes.toBytes("row1");
- private static final byte[] row2 = Bytes.toBytes("row2");
- private static final byte[] initialData = Bytes.toBytes("testWrite-0");
- private static final byte[] data1_q1 = Bytes.toBytes("testWrite-1-q1");
- private static final byte[] data1_q2 = Bytes.toBytes("testWrite-1-q2");
- private static final byte[] data2_q1 = Bytes.toBytes("testWrite-2-q1");
- private static final byte[] data2_q2 = Bytes.toBytes("testWrite-2-q2");
- private static final int TSO1_PORT = 2223;
- private static final int TSO2_PORT = 4321;
-
- private CountDownLatch barrierTillTSOAddressPublication;
-
- private CuratorFramework zkClient;
-
- private TSOServer tso1;
- private TSOServer tso2;
-
- private PausableLeaseManager leaseManager1;
-
- private TransactionManager tm;
-
- @BeforeMethod(alwaysRun = true, timeOut = 30_000)
- public void setup() throws Exception {
- // Get the zkConnection string from minicluster
- String zkConnection = "localhost:" + hBaseUtils.getZkCluster().getClientPort();
-
- zkClient = provideInitializedZookeeperClient(zkConnection);
-
- // Synchronize TSO start
- barrierTillTSOAddressPublication = new CountDownLatch(1);
- final NodeCache currentTSOZNode = new NodeCache(zkClient, CURRENT_TSO_PATH);
- currentTSOZNode.getListenable().addListener(new NodeCacheListener() {
-
- @Override
- public void nodeChanged() throws Exception {
- byte[] currentTSOAndEpochAsBytes = currentTSOZNode.getCurrentData().getData();
- String currentTSOAndEpoch = new String(currentTSOAndEpochAsBytes, Charsets.UTF_8);
- if (currentTSOAndEpoch.endsWith("#0")) { // Wait till a TSO instance publishes the epoch
- barrierTillTSOAddressPublication.countDown();
- }
- }
-
- });
- currentTSOZNode.start(true);
-
- // Configure TSO 1
- TSOServerConfig config1 = new TSOServerConfig();
- config1.setPort(TSO1_PORT);
- config1.setMaxItems(1000);
- config1.setLeaseModule(new TestHALeaseManagementModule(TEST_LEASE_PERIOD_MS, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkConnection, NAMESPACE));
- Injector injector1 = Guice.createInjector(new TestTSOModule(hbaseConf, config1));
- LOG.info("===================== Starting TSO 1 =====================");
- tso1 = injector1.getInstance(TSOServer.class);
- leaseManager1 = (PausableLeaseManager) injector1.getInstance(LeaseManagement.class);
- tso1.startAndWait();
- TestUtils.waitForSocketListening("localhost", TSO1_PORT, 100);
- LOG.info("================ Finished loading TSO 1 ==================");
-
- // Configure TSO 2
- TSOServerConfig config2 = new TSOServerConfig();
- config2.setPort(TSO2_PORT);
- config2.setMaxItems(1000);
- config2.setLeaseModule(new TestHALeaseManagementModule(TEST_LEASE_PERIOD_MS, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkConnection, NAMESPACE));
- Injector injector2 = Guice.createInjector(new TestTSOModule(hbaseConf, config2));
- LOG.info("===================== Starting TSO 2 =====================");
- tso2 = injector2.getInstance(TSOServer.class);
- injector2.getInstance(LeaseManagement.class);
- tso2.startAndWait();
- // Don't do this here: TestUtils.waitForSocketListening("localhost", 4321, 100);
- LOG.info("================ Finished loading TSO 2 ==================");
-
- // Wait till the master TSO is up
- barrierTillTSOAddressPublication.await();
- currentTSOZNode.close();
-
- // Configure HBase TM
- LOG.info("===================== Starting TM =====================");
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionType(HA);
- hbaseOmidClientConf.setConnectionString(zkConnection);
- hbaseOmidClientConf.getOmidClientConfiguration().setZkCurrentTsoPath(CURRENT_TSO_PATH);
- hbaseOmidClientConf.getOmidClientConfiguration().setZkNamespace(NAMESPACE);
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- hbaseConf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 3);
- tm = HBaseTransactionManager.builder(hbaseOmidClientConf).build();
- LOG.info("===================== TM Started =========================");
- }
-
-
- @AfterMethod(alwaysRun = true, timeOut = 60_000)
- public void cleanup() throws Exception {
- LOG.info("Cleanup");
- HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
- deleteTable(admin, TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME));
- hBaseUtils.createTable(Bytes.toBytes(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME),
- new byte[][]{DEFAULT_TIMESTAMP_STORAGE_CF_NAME.getBytes()},
- Integer.MAX_VALUE);
- tso1.stopAndWait();
- TestUtils.waitForSocketNotListening("localhost", TSO1_PORT, 100);
- tso2.stopAndWait();
- TestUtils.waitForSocketNotListening("localhost", TSO2_PORT, 100);
-
- zkClient.delete().forPath(TSO_LEASE_PATH);
- LOG.info("ZKPath {} deleted", TSO_LEASE_PATH);
- zkClient.delete().forPath(CURRENT_TSO_PATH);
- LOG.info("ZKPaths {} deleted", CURRENT_TSO_PATH);
-
- zkClient.close();
- }
-
- //
- // TSO 1 is MASTER & TSO 2 is BACKUP
- // Setup: TX 0 -> Add initial data to cells R1C1 (v0) & R2C2 (v0)
- // TX 1 starts (TSO1)
- // TX 1 modifies cells R1C1 & R2C2 (v1)
- // Interleaved Read TX -IR TX- starts (TSO1)
- // TSO 1 PAUSES -> TSO 2 becomes MASTER
- // IR TX reads R1C1 -> should get v0
- // TX 1 tries to commit -> should abort because was started in TSO 1
- // IR TX reads R2C2 -> should get v0
- // IR TX tries to commit -> should abort because was started in TSO 1
- // End of Test state: R1C1 & R2C2 (v0)
- @Test(timeOut = 60_000)
- public void testScenario1() throws Exception {
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Write initial values for the test
- HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
- long initialEpoch = tx0.getEpoch();
- LOG.info("Starting Tx {} writing initial values for cells ({}) ", tx0, Bytes.toString(initialData));
- Put putInitialDataRow1 = new Put(row1);
- putInitialDataRow1.add(TEST_FAMILY.getBytes(), qualifier1, initialData);
- txTable.put(tx0, putInitialDataRow1);
- Put putInitialDataRow2 = new Put(row2);
- putInitialDataRow2.add(TEST_FAMILY.getBytes(), qualifier2, initialData);
- txTable.put(tx0, putInitialDataRow2);
- tm.commit(tx0);
-
- // Initial checks
- checkRowValues(txTable, initialData, initialData);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx1, Bytes.toString(data1_q1),
- Bytes.toString(data1_q2));
- Put putData1R1Q1 = new Put(row1);
- putData1R1Q1.add(TEST_FAMILY.getBytes(), qualifier1, data1_q1);
- txTable.put(tx1, putData1R1Q1);
- Put putData1R2Q2 = new Put(row2);
- putData1R2Q2.add(TEST_FAMILY.getBytes(), qualifier2, data1_q2);
- txTable.put(tx1, putData1R2Q2);
-
- Transaction interleavedReadTx = tm.begin();
-
- LOG.info("Starting Interleaving Read Tx {} for checking cell values", interleavedReadTx.getTransactionId());
-
- // Simulate a GC pause to change mastership (should throw a ServiceUnavailable exception)
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- LOG.info("++++++++++++++++++++ PAUSING TSO 1 +++++++++++++++++++");
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- leaseManager1.pausedInStillInLeasePeriod();
-
- // Read interleaved and check the values writen by tx 1
- Get getRow1 = new Get(row1).setMaxVersions(1);
- getRow1.addColumn(TEST_FAMILY.getBytes(), qualifier1);
- Result r = txTable.get(interleavedReadTx, getRow1);
- assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), initialData,
- "Unexpected value for SI read R1Q1" + interleavedReadTx + ": "
- + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));
-
- // Try to commit, but it should abort due to the change in mastership
- try {
- tm.commit(tx1);
- fail();
- } catch (RollbackException e) {
- // Expected
- LOG.info("Rollback cause for Tx {}: ", tx1, e.getCause());
- assertEquals(tx1.getStatus(), Transaction.Status.ROLLEDBACK);
- assertEquals(tx1.getEpoch(), initialEpoch);
- }
-
- // Read interleaved and check the values written by tx 1
- Get getRow2 = new Get(row2).setMaxVersions(1);
- r = txTable.get(interleavedReadTx, getRow2);
- assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), initialData,
- "Unexpected value for SI read R2Q2" + interleavedReadTx + ": "
- + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));
-
- // Should commit because its a read only tx does not have to contact the TSO
- tm.commit(interleavedReadTx);
- assertEquals(interleavedReadTx.getEpoch(), initialEpoch);
- assertEquals(interleavedReadTx.getStatus(), Transaction.Status.COMMITTED_RO);
-
- LOG.info("Wait till the client is informed about the connection parameters of the new TSO");
- TestUtils.waitForSocketListening("localhost", TSO2_PORT, 100);
-
- checkRowValues(txTable, initialData, initialData);
-
- // Need to resume to let other test progress
- leaseManager1.resume();
-
- }
-
- }
-
- //
- // TSO 1 is MASTER & TSO 2 is BACKUP
- // Setup: TX 0 -> Add initial data to cells R1C1 (v0) & R2C2 (v0)
- // TX 1 starts (TSO1)
- // TX 1 modifies cells R1C1 & R2C2 (v1)
- // TSO 1 is KILLED -> TSO 2 becomes MASTER
- // TX 1 tries to commit -> should abort because was started in TSO 1
- // TX 2 starts (TSO1)
- // TX 2 reads R1C1 -> should get v0
- // TX 2 reads R2C2 -> should get v0
- // TX 2 modifies cells R1C1 & R2C2 (v2)
- // TX 2 commits
- // End of Test state: R1C1 & R2C2 (v2)
- @Test(timeOut = 60_000)
- public void testScenario2() throws Exception {
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Write initial values for the test
- HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
- long initialEpoch = tx0.getEpoch();
- LOG.info("Starting Tx {} writing initial values for cells ({}) ", tx0, Bytes.toString(initialData));
- Put putInitialDataRow1 = new Put(row1);
- putInitialDataRow1.add(TEST_FAMILY.getBytes(), qualifier1, initialData);
- txTable.put(tx0, putInitialDataRow1);
- Put putInitialDataRow2 = new Put(row2);
- putInitialDataRow2.add(TEST_FAMILY.getBytes(), qualifier2, initialData);
- txTable.put(tx0, putInitialDataRow2);
- tm.commit(tx0);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx1, Bytes.toString(data1_q1),
- Bytes.toString(data1_q2));
- Put putData1R1Q1 = new Put(row1);
- putData1R1Q1.add(TEST_FAMILY.getBytes(), qualifier1, data1_q1);
- txTable.put(tx1, putData1R1Q1);
- Put putData1R2Q2 = new Put(row2);
- putData1R2Q2.add(TEST_FAMILY.getBytes(), qualifier2, data1_q2);
- txTable.put(tx1, putData1R2Q2);
-
- // Provoke change in mastership (should throw a Connection exception)
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- LOG.info("++++++++++++++++++++ KILLING TSO 1 +++++++++++++++++++");
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- LOG.info("++++++++++++++++++++++++++++++++++++++++++++++++++++++");
- tso1.stopAndWait();
- TestUtils.waitForSocketNotListening("localhost", TSO1_PORT, 100);
-
- // Try to commit, but it should abort due to the change in mastership
- try {
- tm.commit(tx1);
- String failMsg = String.format("%s should not commit. Initial epoch was: %d", tx1, initialEpoch);
- fail(failMsg);
- } catch (RollbackException e) {
- // Expected
- LOG.info("Rollback cause for Tx {}: ", tx1, e.getCause());
- assertEquals(tx1.getStatus(), Transaction.Status.ROLLEDBACK);
- assertEquals(tx1.getEpoch(), initialEpoch);
- }
-
- LOG.info("Sleep some time till the client is informed about"
- + "the new TSO connection parameters and how can connect");
- TimeUnit.SECONDS.sleep(10 + 2);
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- LOG.info("Starting Tx {} writing values for cells ({}, {}) ", tx2, Bytes.toString(data1_q1),
- Bytes.toString(data1_q2));
- Get getData1R1Q1 = new Get(row1).setMaxVersions(1);
- Result r = txTable.get(tx2, getData1R1Q1);
- assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), initialData,
- "Unexpected value for SI read R1Q1" + tx2 + ": "
- + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));
- Get getData1R2Q2 = new Get(row2).setMaxVersions(1);
- r = txTable.get(tx2, getData1R2Q2);
- assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), initialData,
- "Unexpected value for SI read R1Q1" + tx2 + ": "
- + Bytes.toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));
-
- Put putData2R1Q1 = new Put(row1);
- putData2R1Q1.add(TEST_FAMILY.getBytes(), qualifier1, data2_q1);
- txTable.put(tx2, putData2R1Q1);
- Put putData2R2Q2 = new Put(row2);
- putData2R2Q2.add(TEST_FAMILY.getBytes(), qualifier2, data2_q2);
- txTable.put(tx2, putData2R2Q2);
- // This one should commit in the new TSO
- tm.commit(tx2);
-
- assertEquals(tx2.getStatus(), Transaction.Status.COMMITTED);
- assertTrue(tx2.getEpoch() > tx0.getCommitTimestamp());
-
- checkRowValues(txTable, data2_q1, data2_q2);
- }
-
- }
-
- private void checkRowValues(TTable txTable, byte[] expectedDataR1Q1, byte[] expectedDataR2Q2)
- throws TransactionException, IOException, RollbackException {
- Transaction readTx = tm.begin();
- LOG.info("Starting Read Tx {} for checking cell values", readTx.getTransactionId());
- Get getRow1 = new Get(row1).setMaxVersions(1);
- getRow1.addColumn(TEST_FAMILY.getBytes(), qualifier1);
- Result r = txTable.get(readTx, getRow1);
- assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier1), expectedDataR1Q1,
- "Unexpected value for SI read R1Q1" + readTx + ": " + Bytes
- .toString(r.getValue(TEST_FAMILY.getBytes(), qualifier1)));
- Get getRow2 = new Get(row2).setMaxVersions(1);
- r = txTable.get(readTx, getRow2);
- assertEquals(r.getValue(TEST_FAMILY.getBytes(), qualifier2), expectedDataR2Q2,
- "Unexpected value for SI read R2Q2" + readTx + ": " + Bytes
- .toString(r.getValue(TEST_FAMILY.getBytes(), qualifier2)));
- tm.commit(readTx);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helpers
- // ----------------------------------------------------------------------------------------------------------------
-
- private static CuratorFramework provideInitializedZookeeperClient(String zkConnection) throws Exception {
-
- LOG.info("Creating Zookeeper Client connecting to {}", zkConnection);
-
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
- CuratorFramework zkClient = CuratorFrameworkFactory
- .builder()
- .namespace(NAMESPACE)
- .connectString(zkConnection)
- .retryPolicy(retryPolicy).build();
-
- LOG.info("Connecting to ZK cluster {}", zkClient.getState());
- zkClient.start();
- zkClient.blockUntilConnected();
- LOG.info("Connection to ZK cluster {}", zkClient.getState());
-
- return zkClient;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
deleted file mode 100644
index 7a3a636..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.ValueFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.spy;
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertNull;
-
-/**
- * Tests to verify that Get and Scan filters still work with transactions tables
- */
-@Test(groups = "sharedHBase")
-public class TestFilters extends OmidTestBase {
-
- byte[] family = Bytes.toBytes(TEST_FAMILY);
- private byte[] row1 = Bytes.toBytes("row1");
- private byte[] row2 = Bytes.toBytes("row2");
- private byte[] row3 = Bytes.toBytes("row3");
- private byte[] prefix = Bytes.toBytes("foo");
- private byte[] col1 = Bytes.toBytes("foobar");
- private byte[] col2 = Bytes.toBytes("boofar");
-
- @Test(timeOut = 60_000)
- public void testGetWithColumnPrefixFilter(ITestContext context) throws Exception {
- testGet(context, new ColumnPrefixFilter(prefix));
- }
-
- @Test(timeOut = 60_000)
- public void testGetWithValueFilter(ITestContext context) throws Exception {
- testGet(context, new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(col1)));
- }
-
- private void testGet(ITestContext context, Filter f) throws Exception {
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString("localhost:1234");
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
-
- TTable table = new TTable(hbaseConf, TEST_TABLE);
- PostCommitActions syncPostCommitter = spy(
- new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- AbstractTransactionManager tm = HBaseTransactionManager.builder(hbaseOmidClientConf)
- .commitTableClient(commitTableClient)
- .postCommitter(syncPostCommitter)
- .build();
-
- writeRows(table, tm, syncPostCommitter);
-
- Transaction t = tm.begin();
- Get g = new Get(row1);
- g.setFilter(f);
-
- Result r = table.get(t, g);
- assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
- assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
-
- g = new Get(row2);
- g.setFilter(f);
- r = table.get(t, g);
- assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
- assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
-
- g = new Get(row3);
- g.setFilter(f);
- r = table.get(t, g);
- assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
- }
-
- @Test(timeOut = 60_000)
- public void testScanWithColumnPrefixFilter(ITestContext context) throws Exception {
- testScan(context, new ColumnPrefixFilter(prefix));
- }
-
- @Test(timeOut = 60_000)
- public void testScanWithValueFilter(ITestContext context) throws Exception {
- testScan(context, new ValueFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(col1)));
- }
-
- private void testScan(ITestContext context, Filter f) throws Exception {
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.getOmidClientConfiguration().setConnectionString("localhost:1234");
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
- PostCommitActions syncPostCommitter = spy(
- new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- AbstractTransactionManager tm = HBaseTransactionManager.builder(hbaseOmidClientConf)
- .commitTableClient(commitTableClient)
- .postCommitter(syncPostCommitter)
- .build();
-
- writeRows(table, tm, syncPostCommitter);
-
- Transaction t = tm.begin();
- Scan s = new Scan().setFilter(f);
-
- ResultScanner rs = table.getScanner(t, s);
-
- Result r = rs.next();
- assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
- assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
-
- r = rs.next();
- assertEquals("should exist in result", 1, r.getColumnCells(family, col1).size());
- assertEquals("shouldn't exist in result", 0, r.getColumnCells(family, col2).size());
-
- r = rs.next();
- assertNull("Last row shouldn't exist", r);
- }
-
-
- private void writeRows(TTable table, TransactionManager tm, PostCommitActions postCommitter)
- throws Exception {
- // create normal row with both cells
- Transaction t = tm.begin();
- Put p = new Put(row1);
- p.add(family, col1, col1);
- p.add(family, col2, col2);
- table.put(t, p);
- tm.commit(t);
-
- // create normal row, but fail to update shadow cells
- doAnswer(new Answer<ListenableFuture<Void>>() {
- public ListenableFuture<Void> answer(InvocationOnMock invocation) {
- // Do not invoke the real method
- return SettableFuture.create();
- }
- }).when(postCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- t = tm.begin();
- p = new Put(row2);
- p.add(family, col1, col1);
- p.add(family, col2, col2);
- table.put(t, p);
- try {
- tm.commit(t);
- } catch (TransactionException e) {
- // Expected, see comment above
- }
-
- // create normal row with only one cell
- t = tm.begin();
- p = new Put(row3);
- p.add(family, col2, col2);
- table.put(t, p);
- try {
- tm.commit(t);
- } catch (TransactionException e) {
- // Expected, see comment above
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
deleted file mode 100644
index 195f4aa..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.timestamp.storage.ZKModule;
-import com.yahoo.omid.tso.LeaseManagement;
-import com.yahoo.omid.tso.Panicker;
-import com.yahoo.omid.tso.PausableLeaseManager;
-import com.yahoo.omid.tso.TSOChannelHandler;
-import com.yahoo.omid.tso.TSOStateManager;
-import org.apache.curator.framework.CuratorFramework;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-class TestHALeaseManagementModule extends AbstractModule {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHALeaseManagementModule.class);
- private final long leasePeriodInMs;
- private final String tsoLeasePath;
- private final String currentTsoPath;
- private final String zkCluster;
- private final String zkNamespace;
-
- TestHALeaseManagementModule(long leasePeriodInMs, String tsoLeasePath, String currentTsoPath,
- String zkCluster, String zkNamespace) {
- this.leasePeriodInMs = leasePeriodInMs;
- this.tsoLeasePath = tsoLeasePath;
- this.currentTsoPath = currentTsoPath;
- this.zkCluster = zkCluster;
- this.zkNamespace = zkNamespace;
- }
-
- @Override
- protected void configure() {
- install(new ZKModule(zkCluster, zkNamespace));
- }
-
- @Provides
- @Singleton
- LeaseManagement provideLeaseManager(@Named(TSO_HOST_AND_PORT_KEY) String tsoHostAndPort,
- TSOChannelHandler tsoChannelHandler,
- TSOStateManager stateManager,
- CuratorFramework zkClient,
- Panicker panicker)
- throws LeaseManagement.LeaseManagementException {
-
- LOG.info("Connection to ZK cluster [{}]", zkClient.getState());
- return new PausableLeaseManager(tsoHostAndPort, tsoChannelHandler, stateManager, leasePeriodInMs,
- tsoLeasePath, currentTsoPath, zkClient, panicker);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
deleted file mode 100644
index 28bfe1e..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.junit.Assert;
-import org.testng.annotations.Test;
-
-public class TestHBaseOmidClientConfiguration {
-
- @Test
- public void testYamlReading() {
- HBaseOmidClientConfiguration configuration = new HBaseOmidClientConfiguration();
- Assert.assertNotNull(configuration.getCommitTableName());
- Assert.assertNotNull(configuration.getHBaseConfiguration());
- Assert.assertNotNull(configuration.getMetrics());
- Assert.assertNotNull(configuration.getOmidClientConfiguration());
- }
-
- @Test
- public void testYamlReadingFromFile() {
- HBaseOmidClientConfiguration configuration = new HBaseOmidClientConfiguration("/test-hbase-omid-client-config.yml");
- Assert.assertNotNull(configuration.getCommitTableName());
- Assert.assertNotNull(configuration.getHBaseConfiguration());
- Assert.assertNotNull(configuration.getMetrics());
- Assert.assertNotNull(configuration.getOmidClientConfiguration());
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
deleted file mode 100644
index 9f3ce1c..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
+++ /dev/null
@@ -1,465 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.transaction.HBaseTransactionManager.CommitTimestampLocatorImpl;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import java.util.Map;
-
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.CACHE;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.COMMIT_TABLE;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.NOT_PRESENT;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.SHADOW_CELL;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.spy;
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestHBaseTransactionClient extends OmidTestBase {
-
- private static final byte[] row1 = Bytes.toBytes("test-is-committed1");
- private static final byte[] row2 = Bytes.toBytes("test-is-committed2");
- private static final byte[] family = Bytes.toBytes(TEST_FAMILY);
- private static final byte[] qualifier = Bytes.toBytes("testdata");
- private static final byte[] data1 = Bytes.toBytes("testWrite-1");
-
- @Test(timeOut = 30_000)
- public void testIsCommitted(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction t1 = (HBaseTransaction) tm.begin();
-
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(t1, put);
- tm.commit(t1);
-
- HBaseTransaction t2 = (HBaseTransaction) tm.begin();
- put = new Put(row2);
- put.add(family, qualifier, data1);
- table.put(t2, put);
- table.getHTable().flushCommits();
-
- HBaseTransaction t3 = (HBaseTransaction) tm.begin();
- put = new Put(row2);
- put.add(family, qualifier, data1);
- table.put(t3, put);
- tm.commit(t3);
-
- HTable htable = new HTable(hbaseConf, TEST_TABLE);
- HBaseCellId hBaseCellId1 = new HBaseCellId(htable, row1, family, qualifier, t1.getStartTimestamp());
- HBaseCellId hBaseCellId2 = new HBaseCellId(htable, row2, family, qualifier, t2.getStartTimestamp());
- HBaseCellId hBaseCellId3 = new HBaseCellId(htable, row2, family, qualifier, t3.getStartTimestamp());
-
- HBaseTransactionClient hbaseTm = (HBaseTransactionClient) newTransactionManager(context);
- assertTrue("row1 should be committed", hbaseTm.isCommitted(hBaseCellId1));
- assertFalse("row2 should not be committed for kv2", hbaseTm.isCommitted(hBaseCellId2));
- assertTrue("row2 should be committed for kv3", hbaseTm.isCommitted(hBaseCellId3));
- }
-
- @Test(timeOut = 30_000)
- public void testCrashAfterCommit(ITestContext context) throws Exception {
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
- AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction t1 = (HBaseTransaction) tm.begin();
-
- // Test shadow cell are created properly
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(t1, put);
- try {
- tm.commit(t1);
- } catch (Exception e) { // (*) crash
- // Do nothing
- }
-
- assertTrue("Cell should be there",
- CellUtils.hasCell(row1,
- family,
- qualifier,
- t1.getStartTimestamp(),
- new TTableCellGetterAdapter(table)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(row1,
- family,
- qualifier,
- t1.getStartTimestamp(),
- new TTableCellGetterAdapter(table)));
-
- HTable htable = new HTable(hbaseConf, TEST_TABLE);
- HBaseCellId hBaseCellId = new HBaseCellId(htable, row1, family, qualifier, t1.getStartTimestamp());
-
- HBaseTransactionClient hbaseTm = (HBaseTransactionClient) newTransactionManager(context);
- assertTrue("row1 should be committed", hbaseTm.isCommitted(hBaseCellId));
- }
-
- @Test(timeOut = 30_000)
- public void testReadCommitTimestampFromCommitTable(ITestContext context) throws Exception {
-
- final long NON_EXISTING_CELL_TS = 1000L;
-
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
- AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- // Test that a non-existing cell timestamp returns an empty result
- Optional<CommitTimestamp> optionalCT = tm.commitTableClient.getCommitTimestamp(NON_EXISTING_CELL_TS).get();
- assertFalse(optionalCT.isPresent());
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
- // Test that we get an invalidation mark for an invalidated transaction
-
- // Start a transaction and invalidate it before commiting it
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
-
- assertTrue(tm.commitTableClient.tryInvalidateTransaction(tx1.getStartTimestamp()).get());
- optionalCT = tm.commitTableClient.getCommitTimestamp(tx1.getStartTimestamp()).get();
- assertTrue(optionalCT.isPresent());
- CommitTimestamp ct = optionalCT.get();
- assertFalse(ct.isValid());
- assertEquals(CommitTable.INVALID_TRANSACTION_MARKER, ct.getValue());
- assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
-
- // Finally test that we get the right commit timestamp for a committed tx
- // that couldn't get
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Put otherPut = new Put(row1);
- otherPut.add(family, qualifier, data1);
- table.put(tx2, otherPut);
- try {
- tm.commit(tx2);
- } catch (Exception e) { // (*) crash
- // Do nothing
- }
-
- optionalCT = tm.commitTableClient.getCommitTimestamp(tx2.getStartTimestamp()).get();
- assertTrue(optionalCT.isPresent());
- ct = optionalCT.get();
- assertTrue(ct.isValid());
- assertEquals(tx2.getCommitTimestamp(), ct.getValue());
- assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
- }
- }
-
- @Test(timeOut = 30_000)
- public void testReadCommitTimestampFromShadowCell(ITestContext context) throws Exception {
-
- final long NON_EXISTING_CELL_TS = 1L;
-
- HBaseTransactionManager tm = (HBaseTransactionManager) newTransactionManager(context);
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Test first we can not found a non-existent cell ts
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier, NON_EXISTING_CELL_TS);
- // Set an empty cache to allow to bypass the checking
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- Optional<CommitTimestamp> optionalCT = tm
- .readCommitTimestampFromShadowCell(NON_EXISTING_CELL_TS, ctLocator);
- assertFalse(optionalCT.isPresent());
-
- // Then test that for a transaction committed, we get the right CT
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
- tm.commit(tx1);
- // Upon commit, the commit data should be in the shadow cells, so test it
- optionalCT = tm.readCommitTimestampFromShadowCell(tx1.getStartTimestamp(), ctLocator);
- assertTrue(optionalCT.isPresent());
- CommitTimestamp ct = optionalCT.get();
- assertTrue(ct.isValid());
- assertEquals(tx1.getCommitTimestamp(), ct.getValue());
- assertTrue(ct.getLocation().compareTo(SHADOW_CELL) == 0);
-
- }
-
- }
-
- // Tests step 1 in AbstractTransactionManager.locateCellCommitTimestamp()
- @Test(timeOut = 30_000)
- public void testCellCommitTimestampIsLocatedInCache(ITestContext context) throws Exception {
-
- final long CELL_ST = 1L;
- final long CELL_CT = 2L;
-
- HBaseTransactionManager tm = (HBaseTransactionManager) newTransactionManager(context);
-
- // Pre-load the element to look for in the cache
- HTable table = new HTable(hbaseConf, TEST_TABLE);
- HBaseCellId hBaseCellId = new HBaseCellId(table, row1, family, qualifier, CELL_ST);
- Map<Long, Long> fakeCache = Maps.newHashMap();
- fakeCache.put(CELL_ST, CELL_CT);
-
- // Then test that locator finds it in the cache
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId, fakeCache);
- CommitTimestamp ct = tm.locateCellCommitTimestamp(CELL_ST, tm.tsoClient.getEpoch(), ctLocator);
- assertTrue(ct.isValid());
- assertEquals(ct.getValue(), CELL_CT);
- assertTrue(ct.getLocation().compareTo(CACHE) == 0);
-
- }
-
- // Tests step 2 in AbstractTransactionManager.locateCellCommitTimestamp()
- // Note: This test is very similar to testCrashAfterCommit() above so
- // maybe we should merge them in this test, adding the missing assertions
- @Test(timeOut = 30_000)
- public void testCellCommitTimestampIsLocatedInCommitTable(ITestContext context) throws Exception {
-
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
- AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
- // Commit a transaction that is broken on commit to avoid
- // write to the shadow cells and avoid cleaning the commit table
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
- try {
- tm.commit(tx1);
- } catch (Exception e) { // (*) crash
- // Do nothing
- }
-
- // Test the locator finds the appropriate data in the commit table
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
- tx1.getStartTimestamp());
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
- ctLocator);
- assertTrue(ct.isValid());
- long expectedCommitTS = tx1.getStartTimestamp() + 1;
- assertEquals(expectedCommitTS, ct.getValue());
- assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
- }
-
- }
-
- // Tests step 3 in AbstractTransactionManager.locateCellCommitTimestamp()
- @Test(timeOut = 30_000)
- public void testCellCommitTimestampIsLocatedInShadowCells(ITestContext context) throws Exception {
-
- HBaseTransactionManager tm = (HBaseTransactionManager) newTransactionManager(context);
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
- // Commit a transaction to add ST/CT in commit table
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
- tm.commit(tx1);
- // Upon commit, the commit data should be in the shadow cells
-
- // Test the locator finds the appropriate data in the shadow cells
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
- tx1.getStartTimestamp());
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
- ctLocator);
- assertTrue(ct.isValid());
- assertEquals(tx1.getCommitTimestamp(), ct.getValue());
- assertTrue(ct.getLocation().compareTo(SHADOW_CELL) == 0);
- }
-
- }
-
- // Tests step 4 in AbstractTransactionManager.locateCellCommitTimestamp()
- @Test(timeOut = 30_000)
- public void testCellFromTransactionInPreviousEpochGetsInvalidComitTimestamp(ITestContext context) throws Exception {
-
- final long CURRENT_EPOCH_FAKE = 1000L;
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
- AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, commitTableClient));
- // The following lines allow to reach step 4)
- // in AbstractTransactionManager.locateCellCommitTimestamp()
- SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
- f.set(Optional.<CommitTimestamp>absent());
- doReturn(f).when(commitTableClient).getCommitTimestamp(any(Long.class));
- doReturn(Optional.<CommitTimestamp>absent()).when(tm).readCommitTimestampFromShadowCell(any(Long.class),
- any(CommitTimestampLocator.class));
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Commit a transaction to add ST/CT in commit table
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
- tm.commit(tx1);
- // Upon commit, the commit data should be in the shadow cells
-
- // Test a transaction in the previous epoch gets an InvalidCommitTimestamp class
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
- tx1.getStartTimestamp());
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- // Fake the current epoch to simulate a newer TSO
- CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), CURRENT_EPOCH_FAKE, ctLocator);
- assertFalse(ct.isValid());
- assertEquals(CommitTable.INVALID_TRANSACTION_MARKER, ct.getValue());
- assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
- }
- }
-
- // Tests step 5 in AbstractTransactionManager.locateCellCommitTimestamp()
- @Test(timeOut = 30_000)
- public void testCellCommitTimestampIsLocatedInCommitTableAfterNotBeingInvalidated(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, syncPostCommitter));
-
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
- // The next two lines avoid steps 2) and 3) and go directly to step 5)
- // in AbstractTransactionManager.locateCellCommitTimestamp()
- SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
- f.set(Optional.<CommitTimestamp>absent());
- doReturn(f).doCallRealMethod().when(commitTableClient).getCommitTimestamp(any(Long.class));
- doReturn(Optional.<CommitTimestamp>absent()).when(tm).readCommitTimestampFromShadowCell(any(Long.class),
- any(CommitTimestampLocator.class));
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Commit a transaction that is broken on commit to avoid
- // write to the shadow cells and avoid cleaning the commit table
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
- try {
- tm.commit(tx1);
- } catch (Exception e) { // (*) crash
- // Do nothing
- }
-
- // Test the locator finds the appropriate data in the commit table
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
- tx1.getStartTimestamp());
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
- ctLocator);
- assertTrue(ct.isValid());
- assertEquals(tx1.getCommitTimestamp(), ct.getValue());
- assertTrue(ct.getLocation().compareTo(COMMIT_TABLE) == 0);
- }
-
- }
-
- // Tests step 6 in AbstractTransactionManager.locateCellCommitTimestamp()
- @Test(timeOut = 30_000)
- public void testCellCommitTimestampIsLocatedInShadowCellsAfterNotBeingInvalidated(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
- AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, commitTableClient));
- // The next two lines avoid steps 2), 3) and 5) and go directly to step 6)
- // in AbstractTransactionManager.locateCellCommitTimestamp()
- SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
- f.set(Optional.<CommitTimestamp>absent());
- doReturn(f).when(commitTableClient).getCommitTimestamp(any(Long.class));
- doReturn(Optional.<CommitTimestamp>absent()).doCallRealMethod()
- .when(tm).readCommitTimestampFromShadowCell(any(Long.class), any(CommitTimestampLocator.class));
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Commit a transaction to add ST/CT in commit table
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(tx1, put);
- tm.commit(tx1);
- // Upon commit, the commit data should be in the shadow cells
-
- // Test the locator finds the appropriate data in the shadow cells
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier,
- tx1.getStartTimestamp());
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- CommitTimestamp ct = tm.locateCellCommitTimestamp(tx1.getStartTimestamp(), tm.tsoClient.getEpoch(),
- ctLocator);
- assertTrue(ct.isValid());
- assertEquals(tx1.getCommitTimestamp(), ct.getValue());
- assertTrue(ct.getLocation().compareTo(SHADOW_CELL) == 0);
- }
-
- }
-
- // Tests last step in AbstractTransactionManager.locateCellCommitTimestamp()
- @Test(timeOut = 30_000)
- public void testCTLocatorReturnsAValidCTWhenNotPresent(ITestContext context) throws Exception {
-
- final long CELL_TS = 1L;
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
- AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, commitTableClient));
- // The following lines allow to reach the last return statement
- SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
- f.set(Optional.<CommitTimestamp>absent());
- doReturn(f).when(commitTableClient).getCommitTimestamp(any(Long.class));
- doReturn(Optional.<CommitTimestamp>absent()).when(tm).readCommitTimestampFromShadowCell(any(Long.class),
- any(CommitTimestampLocator.class));
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
- HBaseCellId hBaseCellId = new HBaseCellId(table.getHTable(), row1, family, qualifier, CELL_TS);
- CommitTimestampLocator ctLocator = new CommitTimestampLocatorImpl(hBaseCellId,
- Maps.<Long, Long>newHashMap());
- CommitTimestamp ct = tm.locateCellCommitTimestamp(CELL_TS, tm.tsoClient.getEpoch(), ctLocator);
- assertTrue(ct.isValid());
- assertEquals(-1L, ct.getValue());
- assertTrue(ct.getLocation().compareTo(NOT_PRESENT) == 0);
- }
- }
-
-}
[30/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
deleted file mode 100644
index 4fe440e..0000000
--- a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
+++ /dev/null
@@ -1,1299 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.yahoo.omid.HBaseShims;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
-import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.spy;
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertNull;
-import static org.testng.AssertJUnit.assertTrue;
-import static org.testng.AssertJUnit.fail;
-
-public class TestCompaction {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class);
-
- private static final String TEST_FAMILY = "test-fam";
- private static final String TEST_QUALIFIER = "test-qual";
-
- private final byte[] fam = Bytes.toBytes(TEST_FAMILY);
- private final byte[] qual = Bytes.toBytes(TEST_QUALIFIER);
- private final byte[] data = Bytes.toBytes("testWrite-1");
-
- private static final int MAX_VERSIONS = 3;
-
- private Random randomGenerator;
- private AbstractTransactionManager tm;
-
- private Injector injector;
-
- private HBaseAdmin admin;
- private Configuration hbaseConf;
- private HBaseTestingUtility hbaseTestUtil;
- private MiniHBaseCluster hbaseCluster;
-
- private TSOServer tso;
-
- private AggregationClient aggregationClient;
- private CommitTable commitTable;
-//<<<<<<< HEAD
-// private TSOClient client;
- private PostCommitActions syncPostCommitter;
-//=======
-//>>>>>>> open-source-master
-
- @BeforeClass
- public void setupTestCompation() throws Exception {
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setPort(1234);
- tsoConfig.setMaxItems(1);
- injector = Guice.createInjector(new TSOForHBaseCompactorTestModule(tsoConfig));
- hbaseConf = injector.getInstance(Configuration.class);
- HBaseCommitTableConfig hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);
- HBaseTimestampStorageConfig hBaseTimestampStorageConfig = injector.getInstance(HBaseTimestampStorageConfig.class);
-
- // settings required for #testDuplicateDeletes()
- hbaseConf.setInt("hbase.hstore.compaction.min", 2);
- hbaseConf.setInt("hbase.hstore.compaction.max", 2);
-
- setupHBase();
- aggregationClient = new AggregationClient(hbaseConf);
- admin = new HBaseAdmin(hbaseConf);
- createRequiredHBaseTables(hBaseTimestampStorageConfig, hBaseCommitTableConfig);
- setupTSO();
-
- commitTable = injector.getInstance(CommitTable.class);
-
-//<<<<<<< HEAD
-// client = TSOClient.newBuilder().withConfiguration(clientConf).build();
-//
-//=======
-//>>>>>>> open-source-master
- }
-
- private void setupHBase() throws Exception {
- LOG.info("********** Setting up HBase **********");
- hbaseTestUtil = new HBaseTestingUtility(hbaseConf);
- LOG.info("********** Creating HBase MiniCluster **********");
- hbaseCluster = hbaseTestUtil.startMiniCluster(1);
- }
-
- private void createRequiredHBaseTables(HBaseTimestampStorageConfig timestampStorageConfig,
- HBaseCommitTableConfig hBaseCommitTableConfig) throws IOException {
- createTableIfNotExists(timestampStorageConfig.getTableName(), timestampStorageConfig.getFamilyName().getBytes());
-
- createTableIfNotExists(hBaseCommitTableConfig.getTableName(), hBaseCommitTableConfig.getCommitTableFamily(), hBaseCommitTableConfig.getLowWatermarkFamily());
- }
-
- private void createTableIfNotExists(String tableName, byte[]... families) throws IOException {
- if (!admin.tableExists(tableName)) {
- LOG.info("Creating {} table...", tableName);
- HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
-
- for (byte[] family : families) {
- HColumnDescriptor datafam = new HColumnDescriptor(family);
- datafam.setMaxVersions(MAX_VERSIONS);
- desc.addFamily(datafam);
- }
-
- desc.addCoprocessor("org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
- admin.createTable(desc);
- for (byte[] family : families) {
- CompactorUtil.enableOmidCompaction(hbaseConf, TableName.valueOf(tableName), family);
- }
- }
-
- }
-
- private void setupTSO() throws IOException, InterruptedException {
- tso = injector.getInstance(TSOServer.class);
- tso.startAndWait();
- TestUtils.waitForSocketListening("localhost", 1234, 100);
- Thread.currentThread().setName("UnitTest(s) thread");
- }
-
- @AfterClass
- public void cleanupTestCompation() throws Exception {
- teardownTSO();
- hbaseCluster.shutdown();
- }
-
- private void teardownTSO() throws IOException, InterruptedException {
- tso.stopAndWait();
- TestUtils.waitForSocketNotListening("localhost", 1234, 1000);
- }
-
- @BeforeMethod
- public void setupTestCompactionIndividualTest() throws Exception {
- randomGenerator = new Random(0xfeedcafeL);
- tm = spy((AbstractTransactionManager) newTransactionManager());
- }
-
- private TransactionManager newTransactionManager() throws Exception {
-//<<<<<<< HEAD
-// CommitTable.Client commitTableClient = commitTable.getClient();
-// syncPostCommitter =
-// spy(new HBaseSyncPostCommitter(new NullMetricsProvider(),commitTableClient));
-//
-// return HBaseTransactionManager.newBuilder()
-// .withConfiguration(hbaseConf)
-// .withCommitTableClient(commitTableClient)
-// .withTSOClient(client)
-// .postCommitter(syncPostCommitter)
-//=======
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString("localhost:1234");
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- CommitTable.Client commitTableClient = commitTable.getClient();
- syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(),commitTableClient));
- return HBaseTransactionManager.builder(hbaseOmidClientConf)
- .postCommitter(syncPostCommitter)
- .commitTableClient(commitTableClient)
-//>>>>>>> open-source-master
- .build();
- }
-
- @Test
- public void testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction() throws Throwable {
- String TEST_TABLE = "testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- final int ROWS_TO_ADD = 5;
-
- long fakeAssignedLowWatermark = 0L;
- for (int i = 0; i < ROWS_TO_ADD; ++i) {
- long rowId = randomGenerator.nextLong();
- Transaction tx = tm.begin();
- if (i == (ROWS_TO_ADD / 2)) {
- fakeAssignedLowWatermark = tx.getTransactionId();
- LOG.info("AssignedLowWatermark " + fakeAssignedLowWatermark);
- }
- Put put = new Put(Bytes.toBytes(rowId));
- put.add(fam, qual, data);
- txTable.put(tx, put);
- tm.commit(tx);
- }
-
- LOG.info("Flushing table {}", TEST_TABLE);
- admin.flush(TEST_TABLE);
-
- // Return a LWM that triggers compaction & stays between 1 and the max start timestamp assigned to previous TXs
- LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
- OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
- .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- CommitTable.Client commitTableClient = spy(commitTable.getClient());
- SettableFuture<Long> f = SettableFuture.create();
- f.set(fakeAssignedLowWatermark);
- doReturn(f).when(commitTableClient).readLowWatermark();
- omidCompactor.commitTableClientQueue.add(commitTableClient);
- LOG.info("Compacting table {}", TEST_TABLE);
- admin.majorCompact(TEST_TABLE);
-
- LOG.info("Sleeping for 3 secs");
- Thread.sleep(3000);
- LOG.info("Waking up after 3 secs");
-
- // No rows should have been discarded after compacting
- assertEquals("Rows in table after compacting should be " + ROWS_TO_ADD, ROWS_TO_ADD, rowCount(TEST_TABLE, fam));
- }
-
- @Test
- public void testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction() throws Exception {
- String TEST_TABLE = "testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- HBaseTransaction problematicTx = (HBaseTransaction) tm.begin();
-
- long row = randomGenerator.nextLong();
-
- // Test shadow cell are created properly
- Put put = new Put(Bytes.toBytes(row));
- put.add(fam, qual, data);
- txTable.put(problematicTx, put);
- try {
- tm.commit(problematicTx);
- } catch (Exception e) { // (*) Crash
- // Do nothing
- }
-
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(row),
- fam,
- qual,
- problematicTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(row),
- fam,
- qual,
- problematicTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
-
- // Return a LWM that triggers compaction and has all the possible start timestamps below it
- LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
- OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
- .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- CommitTable.Client commitTableClient = spy(commitTable.getClient());
- SettableFuture<Long> f = SettableFuture.create();
- f.set(Long.MAX_VALUE);
- doReturn(f).when(commitTableClient).readLowWatermark();
- omidCompactor.commitTableClientQueue.add(commitTableClient);
-
- LOG.info("Flushing table {}", TEST_TABLE);
- admin.flush(TEST_TABLE);
-
- LOG.info("Compacting table {}", TEST_TABLE);
- admin.majorCompact(TEST_TABLE);
-
- LOG.info("Sleeping for 3 secs");
- Thread.sleep(3000);
- LOG.info("Waking up after 3 secs");
-
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(row),
- fam,
- qual,
- problematicTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertTrue("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(row),
- fam,
- qual,
- problematicTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- }
-
- @Test
- public void testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction()
- throws Throwable {
- String
- TEST_TABLE =
- "testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // The KV in this transaction should be discarded
- HBaseTransaction neverendingTxBelowLowWatermark = (HBaseTransaction) tm.begin();
- long rowId = randomGenerator.nextLong();
- Put put = new Put(Bytes.toBytes(rowId));
- put.add(fam, qual, data);
- txTable.put(neverendingTxBelowLowWatermark, put);
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTxBelowLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTxBelowLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
-
- // The KV in this transaction should be added without the shadow cells
- HBaseTransaction neverendingTxAboveLowWatermark = (HBaseTransaction) tm.begin();
- long anotherRowId = randomGenerator.nextLong();
- put = new Put(Bytes.toBytes(anotherRowId));
- put.add(fam, qual, data);
- txTable.put(neverendingTxAboveLowWatermark, put);
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(anotherRowId),
- fam,
- qual,
- neverendingTxAboveLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId),
- fam,
- qual,
- neverendingTxAboveLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
-
- assertEquals("Rows in table before flushing should be 2", 2, rowCount(TEST_TABLE, fam));
- LOG.info("Flushing table {}", TEST_TABLE);
- admin.flush(TEST_TABLE);
- assertEquals("Rows in table after flushing should be 2", 2, rowCount(TEST_TABLE, fam));
-
- // Return a LWM that triggers compaction and stays between both ST of TXs, so assign 1st TX's start timestamp
- LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
- OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
- .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- CommitTable.Client commitTableClient = spy(commitTable.getClient());
- SettableFuture<Long> f = SettableFuture.create();
- f.set(neverendingTxBelowLowWatermark.getStartTimestamp());
- doReturn(f).when(commitTableClient).readLowWatermark();
- omidCompactor.commitTableClientQueue.add(commitTableClient);
- LOG.info("Compacting table {}", TEST_TABLE);
- admin.majorCompact(TEST_TABLE);
-
- LOG.info("Sleeping for 3 secs");
- Thread.sleep(3000);
- LOG.info("Waking up after 3 secs");
-
- // One row should have been discarded after compacting
- assertEquals("There should be only one row in table after compacting", 1, rowCount(TEST_TABLE, fam));
- // The row from the TX below the LWM should not be there (nor its Shadow Cell)
- assertFalse("Cell should not be there",
- CellUtils.hasCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTxBelowLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTxBelowLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- // The row from the TX above the LWM should be there without the Shadow Cell
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(anotherRowId),
- fam,
- qual,
- neverendingTxAboveLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId),
- fam,
- qual,
- neverendingTxAboveLowWatermark.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
-
- }
-
- @Test
- public void testRowsUnalteredWhenCommitTableCannotBeReached() throws Throwable {
- String TEST_TABLE = "testRowsUnalteredWhenCommitTableCannotBeReached";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // The KV in this transaction should be discarded but in the end should remain there because
- // the commit table won't be accessed (simulating an error on access)
- HBaseTransaction neverendingTx = (HBaseTransaction) tm.begin();
- long rowId = randomGenerator.nextLong();
- Put put = new Put(Bytes.toBytes(rowId));
- put.add(fam, qual, data);
- txTable.put(neverendingTx, put);
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
-
- assertEquals("There should be only one rows in table before flushing", 1, rowCount(TEST_TABLE, fam));
- LOG.info("Flushing table {}", TEST_TABLE);
- admin.flush(TEST_TABLE);
- assertEquals("There should be only one rows in table after flushing", 1, rowCount(TEST_TABLE, fam));
-
- // Break access to CommitTable functionality in Compactor
- LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
- OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
- .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- CommitTable.Client commitTableClient = spy(commitTable.getClient());
- SettableFuture<Long> f = SettableFuture.create();
- f.setException(new IOException("Unable to read"));
- doReturn(f).when(commitTableClient).readLowWatermark();
- omidCompactor.commitTableClientQueue.add(commitTableClient);
-
- LOG.info("Compacting table {}", TEST_TABLE);
- admin.majorCompact(TEST_TABLE); // Should trigger the error when accessing CommitTable funct.
-
- LOG.info("Sleeping for 3 secs");
- Thread.sleep(3000);
- LOG.info("Waking up after 3 secs");
-
- // All rows should be there after the failed compaction
- assertEquals("There should be only one row in table after compacting", 1, rowCount(TEST_TABLE, fam));
- assertTrue("Cell should be there",
- CellUtils.hasCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- assertFalse("Shadow cell should not be there",
- CellUtils.hasShadowCell(Bytes.toBytes(rowId),
- fam,
- qual,
- neverendingTx.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- }
-
- @Test
- public void testOriginalTableParametersAreAvoidedAlsoWhenCompacting() throws Throwable {
- String TEST_TABLE = "testOriginalTableParametersAreAvoidedAlsoWhenCompacting";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- long rowId = randomGenerator.nextLong();
- for (int versionCount = 0; versionCount <= (2 * MAX_VERSIONS); versionCount++) {
- Transaction tx = tm.begin();
- Put put = new Put(Bytes.toBytes(rowId));
- put.add(fam, qual, Bytes.toBytes("testWrite-" + versionCount));
- txTable.put(tx, put);
- tm.commit(tx);
- }
-
- Transaction tx = tm.begin();
- Get get = new Get(Bytes.toBytes(rowId));
- get.setMaxVersions(2 * MAX_VERSIONS);
- assertEquals("Max versions should be set to " + (2 * MAX_VERSIONS), (2 * MAX_VERSIONS), get.getMaxVersions());
- get.addColumn(fam, qual);
- Result result = txTable.get(tx, get);
- tm.commit(tx);
- List<Cell> column = result.getColumnCells(fam, qual);
- assertEquals("There should be only one version in the result", 1, column.size());
-
- assertEquals("There should be only one row in table before flushing", 1, rowCount(TEST_TABLE, fam));
- LOG.info("Flushing table {}", TEST_TABLE);
- admin.flush(TEST_TABLE);
- assertEquals("There should be only one row in table after flushing", 1, rowCount(TEST_TABLE, fam));
-
- // Return a LWM that triggers compaction
- compactEverything(TEST_TABLE);
-
- // One row should have been discarded after compacting
- assertEquals("There should be only one row in table after compacting", 1, rowCount(TEST_TABLE, fam));
-
- tx = tm.begin();
- get = new Get(Bytes.toBytes(rowId));
- get.setMaxVersions(2 * MAX_VERSIONS);
- assertEquals("Max versions should be set to " + (2 * MAX_VERSIONS), (2 * MAX_VERSIONS), get.getMaxVersions());
- get.addColumn(fam, qual);
- result = txTable.get(tx, get);
- tm.commit(tx);
- column = result.getColumnCells(fam, qual);
- assertEquals("There should be only one version in the result", 1, column.size());
- assertEquals("Values don't match",
- "testWrite-" + (2 * MAX_VERSIONS),
- Bytes.toString(CellUtil.cloneValue(column.get(0))));
- }
-
- // manually flush the regions on the region server.
- // flushing like this prevents compaction running
- // directly after the flush, which we want to avoid.
- private void manualFlush(String tableName) throws Throwable {
- LOG.info("Manually flushing all regions and waiting 2 secs");
- HBaseShims.flushAllOnlineRegions(hbaseTestUtil.getHBaseCluster().getRegionServer(0),
- TableName.valueOf(tableName));
- TimeUnit.SECONDS.sleep(2);
- }
-
- @Test
- public void testOldCellsAreDiscardedAfterCompaction() throws Exception {
- String TEST_TABLE = "testOldCellsAreDiscardedAfterCompaction";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- byte[] rowId = Bytes.toBytes("row");
-
- // Create 3 transactions modifying the same cell in a particular row
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put1 = new Put(rowId);
- put1.add(fam, qual, Bytes.toBytes("testValue 1"));
- txTable.put(tx1, put1);
- tm.commit(tx1);
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Put put2 = new Put(rowId);
- put2.add(fam, qual, Bytes.toBytes("testValue 2"));
- txTable.put(tx2, put2);
- tm.commit(tx2);
-
- HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
- Put put3 = new Put(rowId);
- put3.add(fam, qual, Bytes.toBytes("testValue 3"));
- txTable.put(tx3, put3);
- tm.commit(tx3);
-
- // Before compaction, the three timestamped values for the cell should be there
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertTrue("Put cell of Tx1 should be there",
- CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
- assertTrue("Put shadow cell of Tx1 should be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
- assertTrue("Put cell of Tx2 cell should be there",
- CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
- assertTrue("Put shadow cell of Tx2 should be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
- assertTrue("Put cell of Tx3 cell should be there",
- CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
- assertTrue("Put shadow cell of Tx3 should be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
-
- // Compact
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- // After compaction, only the last value for the cell should have survived
- assertFalse("Put cell of Tx1 should not be there",
- CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
- assertFalse("Put shadow cell of Tx1 should not be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
- assertFalse("Put cell of Tx2 should not be there",
- CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
- assertFalse("Put shadow cell of Tx2 should not be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
- assertTrue("Put cell of Tx3 cell should be there",
- CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
- assertTrue("Put shadow cell of Tx3 should be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
-
- // A new transaction after compaction should read the last value written
- HBaseTransaction newTx1 = (HBaseTransaction) tm.begin();
- Get newGet1 = new Get(rowId);
- newGet1.addColumn(fam, qual);
- Result result = txTable.get(newTx1, newGet1);
- assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
- // Write a new value
- Put newPut1 = new Put(rowId);
- newPut1.add(fam, qual, Bytes.toBytes("new testValue 1"));
- txTable.put(newTx1, newPut1);
-
- // Start a second new transaction
- HBaseTransaction newTx2 = (HBaseTransaction) tm.begin();
- // Commit first of the new tx
- tm.commit(newTx1);
-
- // The second transaction should still read the previous value
- Get newGet2 = new Get(rowId);
- newGet2.addColumn(fam, qual);
- result = txTable.get(newTx2, newGet2);
- assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
- tm.commit(newTx2);
-
- // Only two values -the new written by newTx1 and the last value
- // for the cell after compaction- should have survived
- assertFalse("Put cell of Tx1 should not be there",
- CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
- assertFalse("Put shadow cell of Tx1 should not be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
- assertFalse("Put cell of Tx2 should not be there",
- CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
- assertFalse("Put shadow cell of Tx2 should not be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
- assertTrue("Put cell of Tx3 cell should be there",
- CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
- assertTrue("Put shadow cell of Tx3 should be there",
- CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
- assertTrue("Put cell of NewTx1 cell should be there",
- CellUtils.hasCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter));
- assertTrue("Put shadow cell of NewTx1 should be there",
- CellUtils.hasShadowCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter));
- }
-
- /**
- * Tests a case where a temporary failure to flush causes the compactor to crash
- */
- @Test
- public void testDuplicateDeletes() throws Throwable {
- String TEST_TABLE = "testDuplicateDeletes";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // jump through hoops to trigger a minor compaction.
- // a minor compaction will only run if there are enough
- // files to be compacted, but that is less than the number
- // of total files, in which case it will run a major
- // compaction. The issue this is testing only shows up
- // with minor compaction, as only Deletes can be duplicate
- // and major compactions filter them out.
- byte[] firstRow = "FirstRow".getBytes();
- HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
- Put put0 = new Put(firstRow);
- put0.add(fam, qual, Bytes.toBytes("testWrite-1"));
- txTable.put(tx0, put0);
- tm.commit(tx0);
-
- // create the first hfile
- manualFlush(TEST_TABLE);
-
- // write a row, it won't be committed
- byte[] rowToBeCompactedAway = "compactMe".getBytes();
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put put1 = new Put(rowToBeCompactedAway);
- put1.add(fam, qual, Bytes.toBytes("testWrite-1"));
- txTable.put(tx1, put1);
- txTable.flushCommits();
-
- // write a row to trigger the double delete problem
- byte[] row = "iCauseErrors".getBytes();
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Put put2 = new Put(row);
- put2.add(fam, qual, Bytes.toBytes("testWrite-1"));
- txTable.put(tx2, put2);
- tm.commit(tx2);
-
- HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
- Put put3 = new Put(row);
- put3.add(fam, qual, Bytes.toBytes("testWrite-1"));
- txTable.put(tx3, put3);
- txTable.flushCommits();
-
- // cause a failure on HBaseTM#preCommit();
- Set<HBaseCellId> writeSet = tx3.getWriteSet();
- assertEquals(1, writeSet.size());
- List<HBaseCellId> newWriteSet = new ArrayList<>();
- final AtomicBoolean flushFailing = new AtomicBoolean(true);
- for (HBaseCellId id : writeSet) {
- HTableInterface failableHTable = spy(id.getTable());
- doAnswer(new Answer<Void>() {
- @Override
- public Void answer(InvocationOnMock invocation)
- throws Throwable {
- if (flushFailing.get()) {
- throw new RetriesExhaustedWithDetailsException(new ArrayList<Throwable>(),
- new ArrayList<Row>(), new ArrayList<String>());
- } else {
- invocation.callRealMethod();
- }
- return null;
- }
- }).when(failableHTable).flushCommits();
-
- newWriteSet.add(new HBaseCellId(failableHTable,
- id.getRow(), id.getFamily(),
- id.getQualifier(), id.getTimestamp()));
- }
- writeSet.clear();
- writeSet.addAll(newWriteSet);
-
- try {
- tm.commit(tx3);
- fail("Shouldn't succeed");
- } catch (TransactionException tme) {
- flushFailing.set(false);
- tm.rollback(tx3);
- }
-
- // create second hfile,
- // it should contain multiple deletes
- manualFlush(TEST_TABLE);
-
- // create loads of files
- byte[] anotherRow = "someotherrow".getBytes();
- HBaseTransaction tx4 = (HBaseTransaction) tm.begin();
- Put put4 = new Put(anotherRow);
- put4.add(fam, qual, Bytes.toBytes("testWrite-1"));
- txTable.put(tx4, put4);
- tm.commit(tx4);
-
- // create third hfile
- manualFlush(TEST_TABLE);
-
- // trigger minor compaction and give it time to run
- setCompactorLWM(tx4.getStartTimestamp(), TEST_TABLE);
- admin.compact(TEST_TABLE);
- Thread.sleep(3000);
-
- // check if the cell that should be compacted, is compacted
- assertFalse("Cell should not be be there",
- CellUtils.hasCell(rowToBeCompactedAway, fam, qual,
- tx1.getStartTimestamp(),
- new TTableCellGetterAdapter(txTable)));
- }
-
- @Test(timeOut = 60000)
- public void testNonOmidCFIsUntouched() throws Throwable {
- String TEST_TABLE = "testNonOmidCFIsUntouched";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- admin.disableTable(TEST_TABLE);
- byte[] nonOmidCF = Bytes.toBytes("nonOmidCF");
- byte[] nonOmidQual = Bytes.toBytes("nonOmidCol");
- HColumnDescriptor nonomidfam = new HColumnDescriptor(nonOmidCF);
- nonomidfam.setMaxVersions(MAX_VERSIONS);
- admin.addColumn(TEST_TABLE, nonomidfam);
- admin.enableTable(TEST_TABLE);
-
- byte[] rowId = Bytes.toBytes("testRow");
- Transaction tx = tm.begin();
- Put put = new Put(rowId);
- put.add(fam, qual, Bytes.toBytes("testValue"));
- txTable.put(tx, put);
-
- Put nonTxPut = new Put(rowId);
- nonTxPut.add(nonOmidCF, nonOmidQual, Bytes.toBytes("nonTxVal"));
- txTable.getHTable().put(nonTxPut);
- txTable.flushCommits(); // to make sure it left the client
-
- Get g = new Get(rowId);
- Result result = txTable.getHTable().get(g);
- assertEquals("Should be there, precompact",
- 1, result.getColumnCells(nonOmidCF, nonOmidQual).size());
- assertEquals("Should be there, precompact",
- 1, result.getColumnCells(fam, qual).size());
-
- compactEverything(TEST_TABLE);
-
- result = txTable.getHTable().get(g);
- assertEquals("Should be there, postcompact",
- 1, result.getColumnCells(nonOmidCF, nonOmidQual).size());
- assertEquals("Should not be there, postcompact",
- 0, result.getColumnCells(fam, qual).size());
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Tests on tombstones and non-transactional Deletes
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * Test that when a major compaction runs, cells that were deleted non-transactionally dissapear
- */
- @Test(timeOut = 60_000)
- public void testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs() throws Throwable {
- String TEST_TABLE = "testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HTable table = new HTable(hbaseConf, TEST_TABLE);
-
- // Write first a value transactionally
- HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("row1");
- Put p0 = new Put(rowId);
- p0.add(fam, qual, Bytes.toBytes("testValue-0"));
- txTable.put(tx0, p0);
- tm.commit(tx0);
-
- // Then perform a non-transactional Delete
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- table.delete(d);
-
- // Trigger a major compaction
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- // Then perform a non-tx (raw) scan...
- Scan scan = new Scan();
- scan.setRaw(true);
- ResultScanner scannerResults = table.getScanner(scan);
-
- // ...and test the deleted cell is not there anymore
- assertNull("There should be no results in scan results", scannerResults.next());
-
- table.close();
-
- }
-
- /**
- * Test that when a minor compaction runs, cells that were deleted non-transactionally are preserved. This is to
- * allow users still access the cells when doing "improper" operations on a transactional table
- */
- @Test(timeOut = 60_000)
- public void testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs() throws Throwable {
- String TEST_TABLE = "testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HTable table = new HTable(hbaseConf, TEST_TABLE);
-
- // Configure the environment to create a minor compaction
-
- // Write first a value transactionally
- HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("row1");
- Put p0 = new Put(rowId);
- p0.add(fam, qual, Bytes.toBytes("testValue-0"));
- txTable.put(tx0, p0);
- tm.commit(tx0);
-
- // create the first hfile
- manualFlush(TEST_TABLE);
-
- // Write another value transactionally
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put p1 = new Put(rowId);
- p1.add(fam, qual, Bytes.toBytes("testValue-1"));
- txTable.put(tx1, p1);
- tm.commit(tx1);
-
- // create the second hfile
- manualFlush(TEST_TABLE);
-
- // Write yet another value transactionally
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Put p2 = new Put(rowId);
- p2.add(fam, qual, Bytes.toBytes("testValue-2"));
- txTable.put(tx2, p2);
- tm.commit(tx2);
-
- // create a third hfile
- manualFlush(TEST_TABLE);
-
- // Then perform a non-transactional Delete
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- table.delete(d);
-
- // create the fourth hfile
- manualFlush(TEST_TABLE);
-
- // Trigger the minor compaction
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
- admin.compact(TEST_TABLE);
- Thread.sleep(5000);
-
- // Then perform a non-tx (raw) scan...
- Scan scan = new Scan();
- scan.setRaw(true);
- ResultScanner scannerResults = table.getScanner(scan);
-
- // ...and test the deleted cell is still there
- int count = 0;
- Result scanResult;
- List<Cell> listOfCellsScanned = new ArrayList<>();
- while ((scanResult = scannerResults.next()) != null) {
- listOfCellsScanned = scanResult.listCells(); // equivalent to rawCells()
- count++;
- }
- assertEquals("There should be only one result in scan results", 1, count);
- assertEquals("There should be three cell entries in the scan results (2 puts, 1 delete)", 3,
- listOfCellsScanned.size());
- boolean wasDeletedCellFound = false;
- int numberOfDeletedCellsFound = 0;
- for (Cell cell : listOfCellsScanned) {
- if (CellUtil.isDelete(cell)) {
- wasDeletedCellFound = true;
- numberOfDeletedCellsFound++;
- }
- }
- assertTrue("We should have found a non-transactionally deleted cell", wasDeletedCellFound);
- assertEquals("There should be only only one deleted cell", 1, numberOfDeletedCellsFound);
-
- table.close();
- }
-
- /**
- * Test that when a minor compaction runs, tombstones are not cleaned up
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs() throws Throwable {
- String TEST_TABLE = "testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // Configure the environment to create a minor compaction
-
- HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("case1");
- Put p = new Put(rowId);
- p.add(fam, qual, Bytes.toBytes("testValue-0"));
- txTable.put(tx0, p);
- tm.commit(tx0);
-
- // create the first hfile
- manualFlush(TEST_TABLE);
-
- // Create the tombstone
- HBaseTransaction deleteTx = (HBaseTransaction) tm.begin();
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(deleteTx, d);
- tm.commit(deleteTx);
-
- // create the second hfile
- manualFlush(TEST_TABLE);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Put p1 = new Put(rowId);
- p1.add(fam, qual, Bytes.toBytes("testValue-11"));
- txTable.put(tx1, p1);
- tm.commit(tx1);
-
- // create the third hfile
- manualFlush(TEST_TABLE);
-
- HBaseTransaction lastTx = (HBaseTransaction) tm.begin();
- Put p2 = new Put(rowId);
- p2.add(fam, qual, Bytes.toBytes("testValue-222"));
- txTable.put(lastTx, p2);
- tm.commit(lastTx);
-
- // Trigger the minor compaction
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
- admin.compact(TEST_TABLE);
- Thread.sleep(5000);
-
- // Checks on results after compaction
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertFalse("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx0.getStartTimestamp(), getter));
- assertFalse("Put shadow cell should be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx0.getStartTimestamp(), getter));
- assertTrue("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Put shadow cell should be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Delete cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- deleteTx.getStartTimestamp(), getter));
- assertTrue("Delete shadow cell should be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- deleteTx.getStartTimestamp(), getter));
- assertTrue("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- lastTx.getStartTimestamp(), getter));
- assertTrue("Put shadow cell should be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- lastTx.getStartTimestamp(), getter));
- }
-
-
- /**
- * Test that when compaction runs, tombstones are cleaned up case1: 1 put (ts < lwm) then tombstone (ts > lwm)
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreCleanedUpCase1() throws Exception {
- String TEST_TABLE = "testTombstonesAreCleanedUpCase1";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("case1");
- Put p = new Put(rowId);
- p.add(fam, qual, Bytes.toBytes("testValue"));
- txTable.put(tx1, p);
- tm.commit(tx1);
-
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(tx2, d);
- tm.commit(tx2);
-
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertTrue("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Put shadow cell should be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Delete cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- assertTrue("Delete shadow cell should be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- }
-
- /**
- * Test that when compaction runs, tombstones are cleaned up case2: 1 put (ts < lwm) then tombstone (ts < lwm)
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreCleanedUpCase2() throws Exception {
- String TEST_TABLE = "testTombstonesAreCleanedUpCase2";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("case2");
- Put p = new Put(rowId);
- p.add(fam, qual, Bytes.toBytes("testValue"));
- txTable.put(tx1, p);
- tm.commit(tx1);
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(tx2, d);
- tm.commit(tx2);
-
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertFalse("Put cell shouldn't be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertFalse("Put shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertFalse("Delete cell shouldn't be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- assertFalse("Delete shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- }
-
- /**
- * Test that when compaction runs, tombstones are cleaned up case3: 1 put (ts < lwm) then tombstone (ts < lwm) not
- * committed
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreCleanedUpCase3() throws Exception {
- String TEST_TABLE = "testTombstonesAreCleanedUpCase3";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("case3");
- Put p = new Put(rowId);
- p.add(fam, qual, Bytes.toBytes("testValue"));
- txTable.put(tx1, p);
- tm.commit(tx1);
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(tx2, d);
-
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertTrue("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Put shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertFalse("Delete cell shouldn't be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- assertFalse("Delete shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- }
-
- /**
- * Test that when compaction runs, tombstones are cleaned up case4: 1 put (ts < lwm) then tombstone (ts > lwm) not
- * committed
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreCleanedUpCase4() throws Exception {
- String TEST_TABLE = "testTombstonesAreCleanedUpCase4";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("case4");
- Put p = new Put(rowId);
- p.add(fam, qual, Bytes.toBytes("testValue"));
- txTable.put(tx1, p);
- tm.commit(tx1);
-
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(tx2, d);
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertTrue("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Put shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Delete cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- assertFalse("Delete shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- }
-
- /**
- * Test that when compaction runs, tombstones are cleaned up case5: tombstone (ts < lwm)
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreCleanedUpCase5() throws Exception {
- String TEST_TABLE = "testTombstonesAreCleanedUpCase5";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- byte[] rowId = Bytes.toBytes("case5");
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(tx1, d);
- tm.commit(tx1);
-
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertFalse("Delete cell shouldn't be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertFalse("Delete shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- }
-
- /**
- * Test that when compaction runs, tombstones are cleaned up case6: tombstone (ts < lwm), then put (ts < lwm)
- */
- @Test(timeOut = 60_000)
- public void testTombstonesAreCleanedUpCase6() throws Exception {
- String TEST_TABLE = "testTombstonesAreCleanedUpCase6";
- createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
- byte[] rowId = Bytes.toBytes("case6");
-
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- Delete d = new Delete(rowId);
- d.deleteColumn(fam, qual);
- txTable.delete(tx1, d);
- tm.commit(tx1);
-
- HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
- Put p = new Put(rowId);
- p.add(fam, qual, Bytes.toBytes("testValue"));
- txTable.put(tx2, p);
- tm.commit(tx2);
-
- HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
- compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
-
- TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
- assertFalse("Delete cell shouldn't be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertFalse("Delete shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx1.getStartTimestamp(), getter));
- assertTrue("Put cell should be there",
- CellUtils.hasCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- assertTrue("Put shadow cell shouldn't be there",
- CellUtils.hasShadowCell(rowId, fam, qual,
- tx2.getStartTimestamp(), getter));
- }
-
- private void setCompactorLWM(long lwm, String tableName) throws Exception {
- OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(tableName)).get(0)
- .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- CommitTable.Client commitTableClient = spy(commitTable.getClient());
- SettableFuture<Long> f = SettableFuture.create();
- f.set(lwm);
- doReturn(f).when(commitTableClient).readLowWatermark();
- omidCompactor.commitTableClientQueue.add(commitTableClient);
- }
-
- private void compactEverything(String tableName) throws Exception {
- compactWithLWM(Long.MAX_VALUE, tableName);
- }
-
- private void compactWithLWM(long lwm, String tableName) throws Exception {
- admin.flush(tableName);
-
- LOG.info("Regions in table {}: {}", tableName, hbaseCluster.getRegions(Bytes.toBytes(tableName)).size());
- setCompactorLWM(lwm, tableName);
- LOG.info("Compacting table {}", tableName);
- admin.majorCompact(tableName);
-
- LOG.info("Sleeping for 3 secs");
- Thread.sleep(3000);
- LOG.info("Waking up after 3 secs");
- }
-
- private long rowCount(String tableName, byte[] family) throws Throwable {
- Scan scan = new Scan();
- scan.addFamily(family);
- return aggregationClient.rowCount(TableName.valueOf(tableName), new LongColumnInterpreter(), scan);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
deleted file mode 100644
index d157512..0000000
--- a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.Client;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.CompactorScanner;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import java.util.Queue;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-
-public class TestCompactorScanner {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestCompactorScanner.class);
-
- private static final long TEST_TS = 1L;
-
- @DataProvider(name = "cell-retain-options")
- public Object[][] createCellRetainOptions() {
- return new Object[][]{
- {1, true}, {2, false},
- };
- }
-
- @Test(dataProvider = "cell-retain-options")
- public void testShouldRetainNonTransactionallyDeletedCellMethod(int optionIdx, boolean retainOption)
- throws Exception {
-
- // Create required mocks
- @SuppressWarnings("unchecked")
- ObserverContext<RegionCoprocessorEnvironment> ctx = mock(ObserverContext.class);
- InternalScanner internalScanner = mock(InternalScanner.class);
- CommitTable.Client ctClient = mock(CommitTable.Client.class);
- @SuppressWarnings("unchecked")
- Queue<Client> queue = mock(Queue.class);
- RegionCoprocessorEnvironment rce = mock(RegionCoprocessorEnvironment.class);
- HRegion hRegion = mock(HRegion.class);
- HRegionInfo hRegionInfo = mock(HRegionInfo.class);
- SettableFuture<Long> f = SettableFuture.<Long>create();
-
- // Wire required mock internals
- f.set(TEST_TS);
- when(ctClient.readLowWatermark()).thenReturn(f);
- when(ctx.getEnvironment()).thenReturn(rce);
- when(rce.getRegion()).thenReturn(hRegion);
- when(hRegion.getRegionInfo()).thenReturn(hRegionInfo);
-
- LOG.info("Testing when retain is {}", retainOption);
- try (CompactorScanner scanner = spy(new CompactorScanner(ctx,
- internalScanner,
- ctClient,
- queue,
- false,
- retainOption))) {
-
- // Different cell types to test
- KeyValue regularKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.Put);
- KeyValue deleteKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.Delete);
- KeyValue deleteColumnKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteColumn);
- KeyValue deleteFamilyKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteFamily);
- KeyValue deleteFamilyVersionKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteFamilyVersion);
-
- assertFalse(scanner.shouldRetainNonTransactionallyDeletedCell(regularKV));
- assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteKV));
- assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteColumnKV));
- assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteFamilyKV));
- assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteFamilyVersionKV));
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TSOForHBaseCompactorTestModule.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TSOForHBaseCompactorTestModule.java b/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TSOForHBaseCompactorTestModule.java
new file mode 100644
index 0000000..91ca7f9
--- /dev/null
+++ b/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TSOForHBaseCompactorTestModule.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorage;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.apache.omid.tso.DisruptorModule;
+import org.apache.omid.tso.LeaseManagement;
+import org.apache.omid.tso.MockPanicker;
+import org.apache.omid.tso.NetworkInterfaceUtils;
+import org.apache.omid.tso.Panicker;
+import org.apache.omid.tso.TSOChannelHandler;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.omid.tso.TSOStateManager;
+import org.apache.omid.tso.TSOStateManagerImpl;
+import org.apache.omid.tso.TimestampOracle;
+import org.apache.omid.tso.TimestampOracleImpl;
+import org.apache.omid.tso.VoidLeaseManager;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+import java.io.File;
+import java.io.IOException;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+class TSOForHBaseCompactorTestModule extends AbstractModule {
+
+ private final TSOServerConfig config;
+
+ TSOForHBaseCompactorTestModule(TSOServerConfig config) {
+ this.config = config;
+ }
+
+ @Override
+ protected void configure() {
+
+ bind(TSOChannelHandler.class).in(Singleton.class);
+
+ bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
+
+ bind(Panicker.class).to(MockPanicker.class);
+ // HBase commit table creation
+ bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class);
+ // Timestamp storage creation
+ bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class);
+ bind(TimestampOracle.class).to(TimestampOracleImpl.class).in(Singleton.class);
+
+ // DisruptorConfig
+ install(new DisruptorModule());
+
+ }
+
+ @Provides
+ @Singleton
+ Configuration provideHBaseConfig() throws IOException {
+ Configuration hbaseConf = HBaseConfiguration.create();
+ hbaseConf.setInt("hbase.hregion.memstore.flush.size", 10_000 * 1024);
+ hbaseConf.setInt("hbase.regionserver.nbreservationblocks", 1);
+ hbaseConf.set("tso.host", "localhost");
+ hbaseConf.setInt("tso.port", 1234);
+ hbaseConf.set("hbase.coprocessor.region.classes", "org.apache.omid.transaction.OmidCompactor");
+ final String rootdir = "/tmp/hbase.test.dir/";
+ File rootdirFile = new File(rootdir);
+ FileUtils.deleteDirectory(rootdirFile);
+ hbaseConf.set("hbase.rootdir", rootdir);
+ return hbaseConf;
+ }
+
+ @Provides
+ TSOServerConfig provideTSOServerConfig() {
+ return config;
+ }
+
+ @Provides
+ @Singleton
+ MetricsRegistry provideMetricsRegistry() {
+ return new NullMetricsProvider();
+ }
+
+ @Provides
+ @Singleton
+ LeaseManagement provideLeaseManager(TSOChannelHandler tsoChannelHandler,
+ TSOStateManager stateManager) throws IOException {
+ return new VoidLeaseManager(tsoChannelHandler, stateManager);
+ }
+
+ @Provides
+ @Named(TSO_HOST_AND_PORT_KEY)
+ String provideTSOHostAndPort() throws SocketException, UnknownHostException {
+ return NetworkInterfaceUtils.getTSOHostAndPort(config);
+
+ }
+}
\ No newline at end of file
[32/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
deleted file mode 100644
index 2287afb..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-
-import java.io.IOException;
-
-/**
- * Contains implementations of the KeyGenerator interface
- */
-public class KeyGeneratorImplementations {
-
- public static KeyGenerator defaultKeyGenerator() {
- return new BucketKeyGenerator();
- }
-
- /**
- * This is the used implementation
- */
- public static class BucketKeyGenerator implements KeyGenerator {
-
- @Override
- public byte[] startTimestampToKey(long startTimestamp) throws IOException {
- byte[] bytes = new byte[9];
- bytes[0] = (byte) (startTimestamp & 0x0F);
- bytes[1] = (byte) ((startTimestamp >> 56) & 0xFF);
- bytes[2] = (byte) ((startTimestamp >> 48) & 0xFF);
- bytes[3] = (byte) ((startTimestamp >> 40) & 0xFF);
- bytes[4] = (byte) ((startTimestamp >> 32) & 0xFF);
- bytes[5] = (byte) ((startTimestamp >> 24) & 0xFF);
- bytes[6] = (byte) ((startTimestamp >> 16) & 0xFF);
- bytes[7] = (byte) ((startTimestamp >> 8) & 0xFF);
- bytes[8] = (byte) ((startTimestamp) & 0xFF);
- return bytes;
- }
-
- @Override
- public long keyToStartTimestamp(byte[] key) {
- assert (key.length == 9);
- return ((long) key[1] & 0xFF) << 56
- | ((long) key[2] & 0xFF) << 48
- | ((long) key[3] & 0xFF) << 40
- | ((long) key[4] & 0xFF) << 32
- | ((long) key[5] & 0xFF) << 24
- | ((long) key[6] & 0xFF) << 16
- | ((long) key[7] & 0xFF) << 8
- | ((long) key[8] & 0xFF);
- }
-
- }
-
- public static class FullRandomKeyGenerator implements KeyGenerator {
-
- @Override
- public byte[] startTimestampToKey(long startTimestamp) {
- assert startTimestamp >= 0;
- // 1) Reverse the timestamp to better spread
- long reversedStartTimestamp = Long.reverse(startTimestamp | Long.MIN_VALUE);
- // 2) Convert to a byte array with big endian format
- byte[] bytes = new byte[8];
- bytes[0] = (byte) ((reversedStartTimestamp >> 56) & 0xFF);
- bytes[1] = (byte) ((reversedStartTimestamp >> 48) & 0xFF);
- bytes[2] = (byte) ((reversedStartTimestamp >> 40) & 0xFF);
- bytes[3] = (byte) ((reversedStartTimestamp >> 32) & 0xFF);
- bytes[4] = (byte) ((reversedStartTimestamp >> 24) & 0xFF);
- bytes[5] = (byte) ((reversedStartTimestamp >> 16) & 0xFF);
- bytes[6] = (byte) ((reversedStartTimestamp >> 8) & 0xFF);
- bytes[7] = (byte) ((reversedStartTimestamp) & 0xFE);
- return bytes;
- }
-
- @Override
- public long keyToStartTimestamp(byte[] key) {
- assert (key.length == 8);
- // 1) Convert from big endian each byte
- long startTimestamp = ((long) key[0] & 0xFF) << 56
- | ((long) key[1] & 0xFF) << 48
- | ((long) key[2] & 0xFF) << 40
- | ((long) key[3] & 0xFF) << 32
- | ((long) key[4] & 0xFF) << 24
- | ((long) key[5] & 0xFF) << 16
- | ((long) key[6] & 0xFF) << 8
- | ((long) key[7] & 0xFF);
- // 2) Reverse to obtain the original value
- return Long.reverse(startTimestamp);
- }
- }
-
- public static class BadRandomKeyGenerator implements KeyGenerator {
-
- @Override
- public byte[] startTimestampToKey(long startTimestamp) throws IOException {
- long reversedStartTimestamp = Long.reverse(startTimestamp);
- byte[] bytes = new byte[CodedOutputStream.computeSFixed64SizeNoTag(reversedStartTimestamp)];
- CodedOutputStream cos = CodedOutputStream.newInstance(bytes);
- cos.writeSFixed64NoTag(reversedStartTimestamp);
- cos.flush();
- return bytes;
- }
-
- @Override
- public long keyToStartTimestamp(byte[] key) throws IOException {
- CodedInputStream cis = CodedInputStream.newInstance(key);
- return Long.reverse(cis.readSFixed64());
- }
-
- }
-
- public static class SeqKeyGenerator implements KeyGenerator {
-
- @Override
- public byte[] startTimestampToKey(long startTimestamp) throws IOException {
- // Convert to a byte array with big endian format
- byte[] bytes = new byte[8];
-
- bytes[0] = (byte) ((startTimestamp >> 56) & 0xFF);
- bytes[1] = (byte) ((startTimestamp >> 48) & 0xFF);
- bytes[2] = (byte) ((startTimestamp >> 40) & 0xFF);
- bytes[3] = (byte) ((startTimestamp >> 32) & 0xFF);
- bytes[4] = (byte) ((startTimestamp >> 24) & 0xFF);
- bytes[5] = (byte) ((startTimestamp >> 16) & 0xFF);
- bytes[6] = (byte) ((startTimestamp >> 8) & 0xFF);
- bytes[7] = (byte) ((startTimestamp) & 0xFF);
- return bytes;
- }
-
- @Override
- public long keyToStartTimestamp(byte[] key) {
- assert (key.length == 8);
- // Convert from big endian each byte
- return ((long) key[0] & 0xFF) << 56
- | ((long) key[1] & 0xFF) << 48
- | ((long) key[2] & 0xFF) << 40
- | ((long) key[3] & 0xFF) << 32
- | ((long) key[4] & 0xFF) << 24
- | ((long) key[5] & 0xFF) << 16
- | ((long) key[6] & 0xFF) << 8
- | ((long) key[7] & 0xFF);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/RegionSplitter.java b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/RegionSplitter.java
deleted file mode 100644
index a0a9259..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/RegionSplitter.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-/**
- * This class contains only the required behavior of the original
- * org.apache.hadoop.hbase.util.RegionSplitter class to avoid
- * having a reference to hbase-testing-util, which transitively
- * imports hbase-server causing dependency conflicts for this module.
- */
-public class RegionSplitter {
-
- /**
- * A generic interface for the RegionSplitter code to use for all it's functionality. Note that the original authors
- * of this code use {@link HexStringSplit} to partition their table and set it as default, but provided this for
- * your custom algorithm. To use, create a new derived class from this interface and call
- * {@link RegionSplitter#createPresplitTable} or
- * {@link RegionSplitter#rollingSplit(String, SplitAlgorithm, Configuration)} with the argument splitClassName
- * giving the name of your class.
- */
- public interface SplitAlgorithm {
- /**
- * Split a pre-existing region into 2 regions.
- *
- * @param start
- * first row (inclusive)
- * @param end
- * last row (exclusive)
- * @return the split row to use
- */
- byte[] split(byte[] start, byte[] end);
-
- /**
- * Split an entire table.
- *
- * @param numRegions
- * number of regions to split the table into
- *
- * @throws RuntimeException
- * user input is validated at this time. may throw a runtime exception in response to a parse
- * failure
- * @return array of split keys for the initial regions of the table. The length of the returned array should be
- * numRegions-1.
- */
- byte[][] split(int numRegions);
-
- /**
- * In HBase, the first row is represented by an empty byte array. This might cause problems with your split
- * algorithm or row printing. All your APIs will be passed firstRow() instead of empty array.
- *
- * @return your representation of your first row
- */
- byte[] firstRow();
-
- /**
- * In HBase, the last row is represented by an empty byte array. This might cause problems with your split
- * algorithm or row printing. All your APIs will be passed firstRow() instead of empty array.
- *
- * @return your representation of your last row
- */
- byte[] lastRow();
-
- /**
- * In HBase, the last row is represented by an empty byte array. Set this value to help the split code
- * understand how to evenly divide the first region.
- *
- * @param userInput
- * raw user input (may throw RuntimeException on parse failure)
- */
- void setFirstRow(String userInput);
-
- /**
- * In HBase, the last row is represented by an empty byte array. Set this value to help the split code
- * understand how to evenly divide the last region. Note that this last row is inclusive for all rows sharing
- * the same prefix.
- *
- * @param userInput
- * raw user input (may throw RuntimeException on parse failure)
- */
- void setLastRow(String userInput);
-
- /**
- * @param input
- * user or file input for row
- * @return byte array representation of this row for HBase
- */
- byte[] strToRow(String input);
-
- /**
- * @param row
- * byte array representing a row in HBase
- * @return String to use for debug & file printing
- */
- String rowToStr(byte[] row);
-
- /**
- * @return the separator character to use when storing / printing the row
- */
- String separator();
-
- /**
- * Set the first row
- *
- * @param userInput
- * byte array of the row key.
- */
- void setFirstRow(byte[] userInput);
-
- /**
- * Set the last row
- *
- * @param userInput
- * byte array of the row key.
- */
- void setLastRow(byte[] userInput);
- }
-
- /**
- * @throws IOException
- * if the specified SplitAlgorithm class couldn't be instantiated
- */
- public static SplitAlgorithm newSplitAlgoInstance(Configuration conf,
- String splitClassName) throws IOException {
- Class<?> splitClass;
-
- // For split algorithms builtin to RegionSplitter, the user can specify
- // their simple class name instead of a fully qualified class name.
- if (splitClassName.equals(UniformSplit.class.getSimpleName())) {
- splitClass = UniformSplit.class;
- } else {
- try {
- splitClass = conf.getClassByName(splitClassName);
- } catch (ClassNotFoundException e) {
- throw new IOException("Couldn't load split class " + splitClassName, e);
- }
- if (splitClass == null) {
- throw new IOException("Failed loading split class " + splitClassName);
- }
- if (!SplitAlgorithm.class.isAssignableFrom(splitClass)) {
- throw new IOException(
- "Specified split class doesn't implement SplitAlgorithm");
- }
- }
- try {
- return splitClass.asSubclass(SplitAlgorithm.class).newInstance();
- } catch (Exception e) {
- throw new IOException("Problem loading split algorithm: ", e);
- }
- }
-
- /**
- * A SplitAlgorithm that divides the space of possible keys evenly. Useful when the keys are approximately uniform
- * random bytes (e.g. hashes). Rows are raw byte values in the range <b>00 => FF</b> and are right-padded with zeros
- * to keep the same memcmp() order. This is the natural algorithm to use for a byte[] environment and saves space,
- * but is not necessarily the easiest for readability.
- */
- public static class UniformSplit implements SplitAlgorithm {
- static final byte xFF = (byte) 0xFF;
- byte[] firstRowBytes = ArrayUtils.EMPTY_BYTE_ARRAY;
- byte[] lastRowBytes =
- new byte[]{xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF};
-
- public byte[] split(byte[] start, byte[] end) {
- return Bytes.split(start, end, 1)[1];
- }
-
- @Override
- public byte[][] split(int numRegions) {
- Preconditions.checkArgument(
- Bytes.compareTo(lastRowBytes, firstRowBytes) > 0,
- "last row (%s) is configured less than first row (%s)",
- Bytes.toStringBinary(lastRowBytes),
- Bytes.toStringBinary(firstRowBytes));
-
- byte[][] splits = Bytes.split(firstRowBytes, lastRowBytes, true,
- numRegions - 1);
- Preconditions.checkState(splits != null,
- "Could not split region with given user input: " + this);
-
- // remove endpoints, which are included in the splits list
- return Arrays.copyOfRange(splits, 1, splits.length - 1);
- }
-
- @Override
- public byte[] firstRow() {
- return firstRowBytes;
- }
-
- @Override
- public byte[] lastRow() {
- return lastRowBytes;
- }
-
- @Override
- public void setFirstRow(String userInput) {
- firstRowBytes = Bytes.toBytesBinary(userInput);
- }
-
- @Override
- public void setLastRow(String userInput) {
- lastRowBytes = Bytes.toBytesBinary(userInput);
- }
-
- @Override
- public void setFirstRow(byte[] userInput) {
- firstRowBytes = userInput;
- }
-
- @Override
- public void setLastRow(byte[] userInput) {
- lastRowBytes = userInput;
- }
-
- @Override
- public byte[] strToRow(String input) {
- return Bytes.toBytesBinary(input);
- }
-
- @Override
- public String rowToStr(byte[] row) {
- return Bytes.toStringBinary(row);
- }
-
- @Override
- public String separator() {
- return ",";
- }
-
- @Override
- public String toString() {
- return this.getClass().getSimpleName() + " [" + rowToStr(firstRow())
- + "," + rowToStr(lastRow()) + "]";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java b/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
deleted file mode 100644
index b4fe81d..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tools.hbase;
-
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public final class HBaseLogin {
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseLogin.class);
-
- public static UserGroupInformation loginIfNeeded(SecureHBaseConfig config) throws IOException {
- if (UserGroupInformation.isSecurityEnabled()) {
- LOG.info("Security is enabled, logging in with principal={}, keytab={}",
- config.getPrincipal(), config.getKeytab());
- UserGroupInformation.loginUserFromKeytab(config.getPrincipal(), config.getKeytab());
- }
- return UserGroupInformation.getCurrentUser();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java b/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
deleted file mode 100644
index 8214780..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tools.hbase;
-
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
-
-public class SecureHBaseConfig {
-
- public static final String HBASE_CLIENT_PRINCIPAL_KEY = "hbase.client.principal";
- public static final String HBASE_CLIENT_KEYTAB_KEY = "hbase.client.keytab";
-
- private String principal = "not set";
- private String keytab = "not set";
-
- // ----------------------------------------------------------------------------------------------------------------
- // WARNING: Do not remove getters/setters, needed by snake_yaml!
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getPrincipal() {
- return principal;
- }
-
- @Inject(optional = true)
- @Named(HBASE_CLIENT_PRINCIPAL_KEY)
- public void setPrincipal(String principal) {
- this.principal = principal;
- }
-
- public String getKeytab() {
- return keytab;
- }
-
- @Inject(optional = true)
- @Named(HBASE_CLIENT_KEYTAB_KEY)
- public void setKeytab(String keytab) {
- this.keytab = keytab;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java b/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
deleted file mode 100644
index 7dd0988..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Objects;
-import org.apache.hadoop.hbase.Cell;
-
-public class CellInfo {
-
- private final Cell cell;
- private final Cell shadowCell;
- private final long timestamp;
-
- public CellInfo(Cell cell, Cell shadowCell) {
- // TODO: Use Guava preconditions instead of the assertions
- assert (cell != null && shadowCell != null);
- assert (cell.getTimestamp() == shadowCell.getTimestamp());
- this.cell = cell;
- this.shadowCell = shadowCell;
- this.timestamp = cell.getTimestamp();
- }
-
- public Cell getCell() {
- return cell;
- }
-
- public Cell getShadowCell() {
- return shadowCell;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("ts", timestamp)
- .add("cell", cell)
- .add("shadow cell", shadowCell)
- .toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java b/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
deleted file mode 100644
index 0d599f6..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Objects;
-import com.google.common.base.Objects.ToStringHelper;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-@SuppressWarnings("all")
-public final class CellUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(CellUtils.class);
- static final byte[] SHADOW_CELL_SUFFIX = "\u0080".getBytes(Charsets.UTF_8); // Non printable char (128 ASCII)
- static byte[] DELETE_TOMBSTONE = Bytes.toBytes("__OMID_TOMBSTONE__");
-
- /**
- * Utility interface to get rid of the dependency on HBase server package
- */
- interface CellGetter {
- Result get(Get get) throws IOException;
- }
-
- /**
- * Returns true if the particular cell passed exists in the datastore.
- * @return true if the cell specified exists. false otherwise
- * @throws IOException
- */
- public static boolean hasCell(byte[] row,
- byte[] family,
- byte[] qualifier,
- long version,
- CellGetter cellGetter)
- throws IOException {
- Get get = new Get(row);
- get.addColumn(family, qualifier);
- get.setTimeStamp(version);
-
- Result result = cellGetter.get(get);
-
- return result.containsColumn(family, qualifier);
- }
-
- /**
- * Returns true if the particular cell passed has a corresponding shadow cell in the datastore.
- * @return true if it has a shadow cell. false otherwise.
- * @throws IOException
- */
- public static boolean hasShadowCell(byte[] row,
- byte[] family,
- byte[] qualifier,
- long version,
- CellGetter cellGetter) throws IOException {
- return hasCell(row, family, addShadowCellSuffix(qualifier),
- version, cellGetter);
- }
-
- /**
- * Builds a new qualifier composed of the HBase qualifier passed + the shadow cell suffix.
- * @param qualifierArray
- * the qualifier to be suffixed
- * @param qualOffset
- * the offset where the qualifier starts
- * @param qualLength
- * the qualifier length
- * @return the suffixed qualifier
- */
- public static byte[] addShadowCellSuffix(byte[] qualifierArray, int qualOffset, int qualLength) {
- byte[] result = new byte[qualLength + SHADOW_CELL_SUFFIX.length];
- System.arraycopy(qualifierArray, qualOffset, result, 0, qualLength);
- System.arraycopy(SHADOW_CELL_SUFFIX, 0, result, qualLength, SHADOW_CELL_SUFFIX.length);
- return result;
- }
-
- /**
- * Builds a new qualifier composed of the HBase qualifier passed + the shadow cell suffix.
- * Contains a reduced signature to avoid boilerplate code in client side.
- * @param qualifier
- * the qualifier to be suffixed
- * @return the suffixed qualifier
- */
- public static byte[] addShadowCellSuffix(byte[] qualifier) {
- return addShadowCellSuffix(qualifier, 0, qualifier.length);
- }
-
- /**
- * Builds a new qualifier removing the shadow cell suffix from the
- * passed HBase qualifier.
- * @param qualifier
- * the qualifier to remove the suffix from
- * @param qualOffset
- * the offset where the qualifier starts
- * @param qualLength
- * the qualifier length
- * @return the new qualifier without the suffix
- */
- public static byte[] removeShadowCellSuffix(byte[] qualifier, int qualOffset, int qualLength) {
-
- if (endsWith(qualifier, qualOffset, qualLength, SHADOW_CELL_SUFFIX)) {
- return Arrays.copyOfRange(qualifier,
- qualOffset,
- qualOffset + (qualLength - SHADOW_CELL_SUFFIX.length));
- }
-
- throw new IllegalArgumentException(
- "Can't find shadow cell suffix in qualifier "
- + Bytes.toString(qualifier));
- }
-
- /**
- * Returns the qualifier length removing the shadow cell suffix. In case that que suffix is not found,
- * just returns the length of the qualifier passed.
- * @param qualifier
- * the qualifier to remove the suffix from
- * @param qualOffset
- * the offset where the qualifier starts
- * @param qualLength
- * the qualifier length
- * @return the qualifier length without the suffix
- */
- public static int qualifierLengthFromShadowCellQualifier(byte[] qualifier, int qualOffset, int qualLength) {
-
- if (endsWith(qualifier, qualOffset, qualLength, SHADOW_CELL_SUFFIX)) {
- return qualLength - SHADOW_CELL_SUFFIX.length;
- }
-
- return qualLength;
-
- }
-
- /**
- * Complement to matchingQualifier() methods in HBase's CellUtil.class
- * @param left
- * the cell to compare the qualifier
- * @param qualArray
- * the explicit qualifier array passed
- * @param qualOffset
- * the explicit qualifier offset passed
- * @param qualLen
- * the explicit qualifier length passed
- * @return whether the qualifiers are equal or not
- */
- public static boolean matchingQualifier(final Cell left, final byte[] qualArray, int qualOffset, int qualLen) {
- return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), left.getQualifierLength(),
- qualArray, qualOffset, qualLen);
- }
-
- /**
- * Check that the cell passed meets the requirements for a valid cell identifier with Omid. Basically, users can't:
- * 1) specify a timestamp
- * 2) use a particular suffix in the qualifier
- */
- public static void validateCell(Cell cell, long startTimestamp) {
- // Throw exception if timestamp is set by the user
- if (cell.getTimestamp() != HConstants.LATEST_TIMESTAMP
- && cell.getTimestamp() != startTimestamp) {
- throw new IllegalArgumentException(
- "Timestamp not allowed in transactional user operations");
- }
- // Throw exception if using a non-allowed qualifier
- if (isShadowCell(cell)) {
- throw new IllegalArgumentException(
- "Reserved string used in column qualifier");
- }
- }
-
- /**
- * Returns whether a cell contains a qualifier that is a shadow cell
- * column qualifier or not.
- * @param cell
- * the cell to check if contains the shadow cell qualifier
- * @return whether the cell passed contains a shadow cell qualifier or not
- */
- public static boolean isShadowCell(Cell cell) {
- byte[] qualifier = cell.getQualifierArray();
- int qualOffset = cell.getQualifierOffset();
- int qualLength = cell.getQualifierLength();
-
- return endsWith(qualifier, qualOffset, qualLength, SHADOW_CELL_SUFFIX);
- }
-
- private static boolean endsWith(byte[] value, int offset, int length, byte[] suffix) {
- if (length <= suffix.length) {
- return false;
- }
-
- int suffixOffset = offset + length - suffix.length;
- int result = Bytes.compareTo(value, suffixOffset, suffix.length,
- suffix, 0, suffix.length);
- return result == 0;
- }
-
- /**
- * Returns if a cell is marked as a tombstone.
- * @param cell
- * the cell to check
- * @return whether the cell is marked as a tombstone or not
- */
- public static boolean isTombstone(Cell cell) {
- return CellUtil.matchingValue(cell, DELETE_TOMBSTONE);
- }
-
- /**
- * Returns a new shadow cell created from a particular cell.
- * @param cell
- * the cell to reconstruct the shadow cell from.
- * @param shadowCellValue
- * the value for the new shadow cell created
- * @return the brand-new shadow cell
- */
- public static Cell buildShadowCellFromCell(Cell cell, byte[] shadowCellValue) {
- byte[] shadowCellQualifier = addShadowCellSuffix(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- return new KeyValue(
- cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
- cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
- shadowCellQualifier, 0, shadowCellQualifier.length,
- cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
- shadowCellValue, 0, shadowCellValue.length);
- }
-
- /**
- * Analyzes a list of cells, associating the corresponding shadow cell if present.
- *
- * @param cells
- * the list of cells to classify
- * @return a sorted map associating each cell with its shadow cell
- */
- public static SortedMap<Cell, Optional<Cell>> mapCellsToShadowCells(List<Cell> cells) {
-
- SortedMap<Cell, Optional<Cell>> cellToShadowCellMap
- = new TreeMap<Cell, Optional<Cell>>(new CellComparator());
-
- Map<CellId, Cell> cellIdToCellMap = new HashMap<CellId, Cell>();
- for (Cell cell : cells) {
- if (!isShadowCell(cell)) {
- CellId key = new CellId(cell, false);
- if (cellIdToCellMap.containsKey(key)) {
- // Get the current cell and compare the values
- Cell storedCell = cellIdToCellMap.get(key);
- if (CellUtil.matchingValue(cell, storedCell)) {
- // TODO: Should we check also here the MVCC and swap if its greater???
- // Values are the same, ignore
- } else {
- if (cell.getMvccVersion() > storedCell.getMvccVersion()) { // Swap values
- Optional<Cell> previousValue = cellToShadowCellMap.remove(storedCell);
- Preconditions.checkNotNull(previousValue, "Should contain an Optional<Cell> value");
- cellIdToCellMap.put(key, cell);
- cellToShadowCellMap.put(cell, previousValue);
- } else {
- LOG.warn("Cell {} with an earlier MVCC found. Ignoring...", cell);
- }
- }
- } else {
- cellIdToCellMap.put(key, cell);
- cellToShadowCellMap.put(cell, Optional.<Cell>absent());
- }
- } else {
- CellId key = new CellId(cell, true);
- if (cellIdToCellMap.containsKey(key)) {
- Cell originalCell = cellIdToCellMap.get(key);
- cellToShadowCellMap.put(originalCell, Optional.of(cell));
- } else {
- LOG.trace("Map does not contain key {}", key);
- }
- }
- }
-
- return cellToShadowCellMap;
- }
-
- private static class CellId {
-
- private static final int MIN_BITS = 32;
-
- private final Cell cell;
- private final boolean isShadowCell;
-
- CellId(Cell cell, boolean isShadowCell) {
-
- this.cell = cell;
- this.isShadowCell = isShadowCell;
-
- }
-
- Cell getCell() {
- return cell;
- }
-
- boolean isShadowCell() {
- return isShadowCell;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this)
- return true;
- if (!(o instanceof CellId))
- return false;
- CellId otherCellId = (CellId) o;
- Cell otherCell = otherCellId.getCell();
-
- // Row comparison
- if (!CellUtil.matchingRow(otherCell, cell)) {
- return false;
- }
-
- // Family comparison
- if (!CellUtil.matchingFamily(otherCell, cell)) {
- return false;
- }
-
- // Qualifier comparison
- if (isShadowCell()) {
- int qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- if (!matchingQualifier(otherCell,
- cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength)) {
- return false;
- }
- } else {
- if (!CellUtil.matchingQualifier(otherCell, cell)) {
- return false;
- }
- }
-
- // Timestamp comparison
- return otherCell.getTimestamp() == cell.getTimestamp();
-
- }
-
- @Override
- public int hashCode() {
- Hasher hasher = Hashing.goodFastHash(MIN_BITS).newHasher();
- hasher.putBytes(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
- hasher.putBytes(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
- int qualifierLength = cell.getQualifierLength();
- if (isShadowCell()) { // Update qualifier length when qualifier is shadow cell
- qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- }
- hasher.putBytes(cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength);
- hasher.putLong(cell.getTimestamp());
- return hasher.hash().asInt();
- }
-
- @Override
- public String toString() {
- ToStringHelper helper = Objects.toStringHelper(this);
- helper.add("row", Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
- helper.add("family", Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()));
- helper.add("is shadow cell?", isShadowCell);
- helper.add("qualifier",
- Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
- if (isShadowCell()) {
- int qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- helper.add("qualifier whithout shadow cell suffix",
- Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength));
- }
- helper.add("ts", cell.getTimestamp());
- return helper.toString();
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/HBaseConfigModule.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/HBaseConfigModule.java b/hbase-common/src/main/java/org/apache/omid/HBaseConfigModule.java
new file mode 100644
index 0000000..5c2acdc
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/HBaseConfigModule.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.tools.hbase.HBaseLogin;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+import java.io.IOException;
+
+public class HBaseConfigModule extends AbstractModule {
+
+ private String principal;
+ private String keytab;
+
+ public HBaseConfigModule(String principal, String keytab) {
+ this.principal = principal;
+ this.keytab = keytab;
+ }
+
+ @Override
+ protected void configure() {
+ }
+
+ @Provides
+ public Configuration provideHBaseConfig() throws IOException {
+ Configuration configuration = HBaseConfiguration.create();
+ SecureHBaseConfig secureHBaseConfig = new SecureHBaseConfig();
+ secureHBaseConfig.setKeytab(keytab);
+ secureHBaseConfig.setPrincipal(principal);
+ HBaseLogin.loginIfNeeded(secureHBaseConfig);
+ return configuration;
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // IMPORTANT NOTE
+ // Overriding equals() and hashCode() is necessary here to allow this module to be installed multiple times.
+ // As example, this module can be installed in a configuration using both the HBase Timestamp & Commit Table stores.
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public boolean equals(Object o) {
+
+ if (this == o) return true;
+
+ if (o == null || getClass() != o.getClass()) return false;
+
+ HBaseConfigModule that = (HBaseConfigModule) o;
+
+ if (principal != null ? !principal.equals(that.principal) : that.principal != null) return false;
+ return keytab != null ? keytab.equals(that.keytab) : that.keytab == null;
+
+ }
+
+ @Override
+ public int hashCode() {
+
+ int result = principal != null ? principal.hashCode() : 0;
+ result = 31 * result + (keytab != null ? keytab.hashCode() : 0);
+ return result;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGenerator.java b/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGenerator.java
new file mode 100644
index 0000000..774d432
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGenerator.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import java.io.IOException;
+
+/**
+ * Implementations of this interface determine how keys are spread in HBase
+ */
+public interface KeyGenerator {
+ byte[] startTimestampToKey(long startTimestamp) throws IOException;
+
+ long keyToStartTimestamp(byte[] key) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGeneratorImplementations.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGeneratorImplementations.java b/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGeneratorImplementations.java
new file mode 100644
index 0000000..a0e464c
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/committable/hbase/KeyGeneratorImplementations.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+
+import java.io.IOException;
+
+/**
+ * Contains implementations of the KeyGenerator interface
+ */
+public class KeyGeneratorImplementations {
+
+ public static KeyGenerator defaultKeyGenerator() {
+ return new BucketKeyGenerator();
+ }
+
+ /**
+ * This is the used implementation
+ */
+ public static class BucketKeyGenerator implements KeyGenerator {
+
+ @Override
+ public byte[] startTimestampToKey(long startTimestamp) throws IOException {
+ byte[] bytes = new byte[9];
+ bytes[0] = (byte) (startTimestamp & 0x0F);
+ bytes[1] = (byte) ((startTimestamp >> 56) & 0xFF);
+ bytes[2] = (byte) ((startTimestamp >> 48) & 0xFF);
+ bytes[3] = (byte) ((startTimestamp >> 40) & 0xFF);
+ bytes[4] = (byte) ((startTimestamp >> 32) & 0xFF);
+ bytes[5] = (byte) ((startTimestamp >> 24) & 0xFF);
+ bytes[6] = (byte) ((startTimestamp >> 16) & 0xFF);
+ bytes[7] = (byte) ((startTimestamp >> 8) & 0xFF);
+ bytes[8] = (byte) ((startTimestamp) & 0xFF);
+ return bytes;
+ }
+
+ @Override
+ public long keyToStartTimestamp(byte[] key) {
+ assert (key.length == 9);
+ return ((long) key[1] & 0xFF) << 56
+ | ((long) key[2] & 0xFF) << 48
+ | ((long) key[3] & 0xFF) << 40
+ | ((long) key[4] & 0xFF) << 32
+ | ((long) key[5] & 0xFF) << 24
+ | ((long) key[6] & 0xFF) << 16
+ | ((long) key[7] & 0xFF) << 8
+ | ((long) key[8] & 0xFF);
+ }
+
+ }
+
+ public static class FullRandomKeyGenerator implements KeyGenerator {
+
+ @Override
+ public byte[] startTimestampToKey(long startTimestamp) {
+ assert startTimestamp >= 0;
+ // 1) Reverse the timestamp to better spread
+ long reversedStartTimestamp = Long.reverse(startTimestamp | Long.MIN_VALUE);
+ // 2) Convert to a byte array with big endian format
+ byte[] bytes = new byte[8];
+ bytes[0] = (byte) ((reversedStartTimestamp >> 56) & 0xFF);
+ bytes[1] = (byte) ((reversedStartTimestamp >> 48) & 0xFF);
+ bytes[2] = (byte) ((reversedStartTimestamp >> 40) & 0xFF);
+ bytes[3] = (byte) ((reversedStartTimestamp >> 32) & 0xFF);
+ bytes[4] = (byte) ((reversedStartTimestamp >> 24) & 0xFF);
+ bytes[5] = (byte) ((reversedStartTimestamp >> 16) & 0xFF);
+ bytes[6] = (byte) ((reversedStartTimestamp >> 8) & 0xFF);
+ bytes[7] = (byte) ((reversedStartTimestamp) & 0xFE);
+ return bytes;
+ }
+
+ @Override
+ public long keyToStartTimestamp(byte[] key) {
+ assert (key.length == 8);
+ // 1) Convert from big endian each byte
+ long startTimestamp = ((long) key[0] & 0xFF) << 56
+ | ((long) key[1] & 0xFF) << 48
+ | ((long) key[2] & 0xFF) << 40
+ | ((long) key[3] & 0xFF) << 32
+ | ((long) key[4] & 0xFF) << 24
+ | ((long) key[5] & 0xFF) << 16
+ | ((long) key[6] & 0xFF) << 8
+ | ((long) key[7] & 0xFF);
+ // 2) Reverse to obtain the original value
+ return Long.reverse(startTimestamp);
+ }
+ }
+
+ public static class BadRandomKeyGenerator implements KeyGenerator {
+
+ @Override
+ public byte[] startTimestampToKey(long startTimestamp) throws IOException {
+ long reversedStartTimestamp = Long.reverse(startTimestamp);
+ byte[] bytes = new byte[CodedOutputStream.computeSFixed64SizeNoTag(reversedStartTimestamp)];
+ CodedOutputStream cos = CodedOutputStream.newInstance(bytes);
+ cos.writeSFixed64NoTag(reversedStartTimestamp);
+ cos.flush();
+ return bytes;
+ }
+
+ @Override
+ public long keyToStartTimestamp(byte[] key) throws IOException {
+ CodedInputStream cis = CodedInputStream.newInstance(key);
+ return Long.reverse(cis.readSFixed64());
+ }
+
+ }
+
+ public static class SeqKeyGenerator implements KeyGenerator {
+
+ @Override
+ public byte[] startTimestampToKey(long startTimestamp) throws IOException {
+ // Convert to a byte array with big endian format
+ byte[] bytes = new byte[8];
+
+ bytes[0] = (byte) ((startTimestamp >> 56) & 0xFF);
+ bytes[1] = (byte) ((startTimestamp >> 48) & 0xFF);
+ bytes[2] = (byte) ((startTimestamp >> 40) & 0xFF);
+ bytes[3] = (byte) ((startTimestamp >> 32) & 0xFF);
+ bytes[4] = (byte) ((startTimestamp >> 24) & 0xFF);
+ bytes[5] = (byte) ((startTimestamp >> 16) & 0xFF);
+ bytes[6] = (byte) ((startTimestamp >> 8) & 0xFF);
+ bytes[7] = (byte) ((startTimestamp) & 0xFF);
+ return bytes;
+ }
+
+ @Override
+ public long keyToStartTimestamp(byte[] key) {
+ assert (key.length == 8);
+ // Convert from big endian each byte
+ return ((long) key[0] & 0xFF) << 56
+ | ((long) key[1] & 0xFF) << 48
+ | ((long) key[2] & 0xFF) << 40
+ | ((long) key[3] & 0xFF) << 32
+ | ((long) key[4] & 0xFF) << 24
+ | ((long) key[5] & 0xFF) << 16
+ | ((long) key[6] & 0xFF) << 8
+ | ((long) key[7] & 0xFF);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/committable/hbase/RegionSplitter.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/committable/hbase/RegionSplitter.java b/hbase-common/src/main/java/org/apache/omid/committable/hbase/RegionSplitter.java
new file mode 100644
index 0000000..b821b31
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/committable/hbase/RegionSplitter.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * This class contains only the required behavior of the original
+ * org.apache.hadoop.hbase.util.RegionSplitter class to avoid
+ * having a reference to hbase-testing-util, which transitively
+ * imports hbase-server causing dependency conflicts for this module.
+ */
+public class RegionSplitter {
+
+ /**
+ * A generic interface for the RegionSplitter code to use for all it's functionality. Note that the original authors
+ * of this code use {@link HexStringSplit} to partition their table and set it as default, but provided this for
+ * your custom algorithm. To use, create a new derived class from this interface and call
+ * {@link RegionSplitter#createPresplitTable} or
+ * {@link RegionSplitter#rollingSplit(String, SplitAlgorithm, Configuration)} with the argument splitClassName
+ * giving the name of your class.
+ */
+ public interface SplitAlgorithm {
+ /**
+ * Split a pre-existing region into 2 regions.
+ *
+ * @param start
+ * first row (inclusive)
+ * @param end
+ * last row (exclusive)
+ * @return the split row to use
+ */
+ byte[] split(byte[] start, byte[] end);
+
+ /**
+ * Split an entire table.
+ *
+ * @param numRegions
+ * number of regions to split the table into
+ *
+ * @throws RuntimeException
+ * user input is validated at this time. may throw a runtime exception in response to a parse
+ * failure
+ * @return array of split keys for the initial regions of the table. The length of the returned array should be
+ * numRegions-1.
+ */
+ byte[][] split(int numRegions);
+
+ /**
+ * In HBase, the first row is represented by an empty byte array. This might cause problems with your split
+ * algorithm or row printing. All your APIs will be passed firstRow() instead of empty array.
+ *
+ * @return your representation of your first row
+ */
+ byte[] firstRow();
+
+ /**
+ * In HBase, the last row is represented by an empty byte array. This might cause problems with your split
+ * algorithm or row printing. All your APIs will be passed firstRow() instead of empty array.
+ *
+ * @return your representation of your last row
+ */
+ byte[] lastRow();
+
+ /**
+ * In HBase, the last row is represented by an empty byte array. Set this value to help the split code
+ * understand how to evenly divide the first region.
+ *
+ * @param userInput
+ * raw user input (may throw RuntimeException on parse failure)
+ */
+ void setFirstRow(String userInput);
+
+ /**
+ * In HBase, the last row is represented by an empty byte array. Set this value to help the split code
+ * understand how to evenly divide the last region. Note that this last row is inclusive for all rows sharing
+ * the same prefix.
+ *
+ * @param userInput
+ * raw user input (may throw RuntimeException on parse failure)
+ */
+ void setLastRow(String userInput);
+
+ /**
+ * @param input
+ * user or file input for row
+ * @return byte array representation of this row for HBase
+ */
+ byte[] strToRow(String input);
+
+ /**
+ * @param row
+ * byte array representing a row in HBase
+ * @return String to use for debug & file printing
+ */
+ String rowToStr(byte[] row);
+
+ /**
+ * @return the separator character to use when storing / printing the row
+ */
+ String separator();
+
+ /**
+ * Set the first row
+ *
+ * @param userInput
+ * byte array of the row key.
+ */
+ void setFirstRow(byte[] userInput);
+
+ /**
+ * Set the last row
+ *
+ * @param userInput
+ * byte array of the row key.
+ */
+ void setLastRow(byte[] userInput);
+ }
+
+ /**
+ * @throws IOException
+ * if the specified SplitAlgorithm class couldn't be instantiated
+ */
+ public static SplitAlgorithm newSplitAlgoInstance(Configuration conf,
+ String splitClassName) throws IOException {
+ Class<?> splitClass;
+
+ // For split algorithms builtin to RegionSplitter, the user can specify
+ // their simple class name instead of a fully qualified class name.
+ if (splitClassName.equals(UniformSplit.class.getSimpleName())) {
+ splitClass = UniformSplit.class;
+ } else {
+ try {
+ splitClass = conf.getClassByName(splitClassName);
+ } catch (ClassNotFoundException e) {
+ throw new IOException("Couldn't load split class " + splitClassName, e);
+ }
+ if (splitClass == null) {
+ throw new IOException("Failed loading split class " + splitClassName);
+ }
+ if (!SplitAlgorithm.class.isAssignableFrom(splitClass)) {
+ throw new IOException(
+ "Specified split class doesn't implement SplitAlgorithm");
+ }
+ }
+ try {
+ return splitClass.asSubclass(SplitAlgorithm.class).newInstance();
+ } catch (Exception e) {
+ throw new IOException("Problem loading split algorithm: ", e);
+ }
+ }
+
+ /**
+ * A SplitAlgorithm that divides the space of possible keys evenly. Useful when the keys are approximately uniform
+ * random bytes (e.g. hashes). Rows are raw byte values in the range <b>00 => FF</b> and are right-padded with zeros
+ * to keep the same memcmp() order. This is the natural algorithm to use for a byte[] environment and saves space,
+ * but is not necessarily the easiest for readability.
+ */
+ public static class UniformSplit implements SplitAlgorithm {
+ static final byte xFF = (byte) 0xFF;
+ byte[] firstRowBytes = ArrayUtils.EMPTY_BYTE_ARRAY;
+ byte[] lastRowBytes =
+ new byte[]{xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF};
+
+ public byte[] split(byte[] start, byte[] end) {
+ return Bytes.split(start, end, 1)[1];
+ }
+
+ @Override
+ public byte[][] split(int numRegions) {
+ Preconditions.checkArgument(
+ Bytes.compareTo(lastRowBytes, firstRowBytes) > 0,
+ "last row (%s) is configured less than first row (%s)",
+ Bytes.toStringBinary(lastRowBytes),
+ Bytes.toStringBinary(firstRowBytes));
+
+ byte[][] splits = Bytes.split(firstRowBytes, lastRowBytes, true,
+ numRegions - 1);
+ Preconditions.checkState(splits != null,
+ "Could not split region with given user input: " + this);
+
+ // remove endpoints, which are included in the splits list
+ return Arrays.copyOfRange(splits, 1, splits.length - 1);
+ }
+
+ @Override
+ public byte[] firstRow() {
+ return firstRowBytes;
+ }
+
+ @Override
+ public byte[] lastRow() {
+ return lastRowBytes;
+ }
+
+ @Override
+ public void setFirstRow(String userInput) {
+ firstRowBytes = Bytes.toBytesBinary(userInput);
+ }
+
+ @Override
+ public void setLastRow(String userInput) {
+ lastRowBytes = Bytes.toBytesBinary(userInput);
+ }
+
+ @Override
+ public void setFirstRow(byte[] userInput) {
+ firstRowBytes = userInput;
+ }
+
+ @Override
+ public void setLastRow(byte[] userInput) {
+ lastRowBytes = userInput;
+ }
+
+ @Override
+ public byte[] strToRow(String input) {
+ return Bytes.toBytesBinary(input);
+ }
+
+ @Override
+ public String rowToStr(byte[] row) {
+ return Bytes.toStringBinary(row);
+ }
+
+ @Override
+ public String separator() {
+ return ",";
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getSimpleName() + " [" + rowToStr(firstRow())
+ + "," + rowToStr(lastRow()) + "]";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/tools/hbase/HBaseLogin.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/tools/hbase/HBaseLogin.java b/hbase-common/src/main/java/org/apache/omid/tools/hbase/HBaseLogin.java
new file mode 100644
index 0000000..af9b96c
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/tools/hbase/HBaseLogin.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tools.hbase;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public final class HBaseLogin {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseLogin.class);
+
+ public static UserGroupInformation loginIfNeeded(SecureHBaseConfig config) throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ LOG.info("Security is enabled, logging in with principal={}, keytab={}",
+ config.getPrincipal(), config.getKeytab());
+ UserGroupInformation.loginUserFromKeytab(config.getPrincipal(), config.getKeytab());
+ }
+ return UserGroupInformation.getCurrentUser();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/tools/hbase/SecureHBaseConfig.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/tools/hbase/SecureHBaseConfig.java b/hbase-common/src/main/java/org/apache/omid/tools/hbase/SecureHBaseConfig.java
new file mode 100644
index 0000000..e10a56d
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/tools/hbase/SecureHBaseConfig.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tools.hbase;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
+public class SecureHBaseConfig {
+
+ public static final String HBASE_CLIENT_PRINCIPAL_KEY = "hbase.client.principal";
+ public static final String HBASE_CLIENT_KEYTAB_KEY = "hbase.client.keytab";
+
+ private String principal = "not set";
+ private String keytab = "not set";
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // WARNING: Do not remove getters/setters, needed by snake_yaml!
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ @Inject(optional = true)
+ @Named(HBASE_CLIENT_PRINCIPAL_KEY)
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public String getKeytab() {
+ return keytab;
+ }
+
+ @Inject(optional = true)
+ @Named(HBASE_CLIENT_KEYTAB_KEY)
+ public void setKeytab(String keytab) {
+ this.keytab = keytab;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/transaction/CellInfo.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/transaction/CellInfo.java b/hbase-common/src/main/java/org/apache/omid/transaction/CellInfo.java
new file mode 100644
index 0000000..2c885a4
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/transaction/CellInfo.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Objects;
+import org.apache.hadoop.hbase.Cell;
+
+public class CellInfo {
+
+ private final Cell cell;
+ private final Cell shadowCell;
+ private final long timestamp;
+
+ public CellInfo(Cell cell, Cell shadowCell) {
+ // TODO: Use Guava preconditions instead of the assertions
+ assert (cell != null && shadowCell != null);
+ assert (cell.getTimestamp() == shadowCell.getTimestamp());
+ this.cell = cell;
+ this.shadowCell = shadowCell;
+ this.timestamp = cell.getTimestamp();
+ }
+
+ public Cell getCell() {
+ return cell;
+ }
+
+ public Cell getShadowCell() {
+ return shadowCell;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("ts", timestamp)
+ .add("cell", cell)
+ .add("shadow cell", shadowCell)
+ .toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/org/apache/omid/transaction/CellUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/omid/transaction/CellUtils.java b/hbase-common/src/main/java/org/apache/omid/transaction/CellUtils.java
new file mode 100644
index 0000000..f22ffb5
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/omid/transaction/CellUtils.java
@@ -0,0 +1,406 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Objects;
+import com.google.common.base.Objects.ToStringHelper;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+@SuppressWarnings("all")
+public final class CellUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CellUtils.class);
+ static final byte[] SHADOW_CELL_SUFFIX = "\u0080".getBytes(Charsets.UTF_8); // Non printable char (128 ASCII)
+ static byte[] DELETE_TOMBSTONE = Bytes.toBytes("__OMID_TOMBSTONE__");
+
+ /**
+ * Utility interface to get rid of the dependency on HBase server package
+ */
+ interface CellGetter {
+ Result get(Get get) throws IOException;
+ }
+
+ /**
+ * Returns true if the particular cell passed exists in the datastore.
+ * @return true if the cell specified exists. false otherwise
+ * @throws IOException
+ */
+ public static boolean hasCell(byte[] row,
+ byte[] family,
+ byte[] qualifier,
+ long version,
+ CellGetter cellGetter)
+ throws IOException {
+ Get get = new Get(row);
+ get.addColumn(family, qualifier);
+ get.setTimeStamp(version);
+
+ Result result = cellGetter.get(get);
+
+ return result.containsColumn(family, qualifier);
+ }
+
+ /**
+ * Returns true if the particular cell passed has a corresponding shadow cell in the datastore.
+ * @return true if it has a shadow cell. false otherwise.
+ * @throws IOException
+ */
+ public static boolean hasShadowCell(byte[] row,
+ byte[] family,
+ byte[] qualifier,
+ long version,
+ CellGetter cellGetter) throws IOException {
+ return hasCell(row, family, addShadowCellSuffix(qualifier),
+ version, cellGetter);
+ }
+
+ /**
+ * Builds a new qualifier composed of the HBase qualifier passed + the shadow cell suffix.
+ * @param qualifierArray
+ * the qualifier to be suffixed
+ * @param qualOffset
+ * the offset where the qualifier starts
+ * @param qualLength
+ * the qualifier length
+ * @return the suffixed qualifier
+ */
+ public static byte[] addShadowCellSuffix(byte[] qualifierArray, int qualOffset, int qualLength) {
+ byte[] result = new byte[qualLength + SHADOW_CELL_SUFFIX.length];
+ System.arraycopy(qualifierArray, qualOffset, result, 0, qualLength);
+ System.arraycopy(SHADOW_CELL_SUFFIX, 0, result, qualLength, SHADOW_CELL_SUFFIX.length);
+ return result;
+ }
+
+ /**
+ * Builds a new qualifier composed of the HBase qualifier passed + the shadow cell suffix.
+ * Contains a reduced signature to avoid boilerplate code in client side.
+ * @param qualifier
+ * the qualifier to be suffixed
+ * @return the suffixed qualifier
+ */
+ public static byte[] addShadowCellSuffix(byte[] qualifier) {
+ return addShadowCellSuffix(qualifier, 0, qualifier.length);
+ }
+
+ /**
+ * Builds a new qualifier removing the shadow cell suffix from the
+ * passed HBase qualifier.
+ * @param qualifier
+ * the qualifier to remove the suffix from
+ * @param qualOffset
+ * the offset where the qualifier starts
+ * @param qualLength
+ * the qualifier length
+ * @return the new qualifier without the suffix
+ */
+ public static byte[] removeShadowCellSuffix(byte[] qualifier, int qualOffset, int qualLength) {
+
+ if (endsWith(qualifier, qualOffset, qualLength, SHADOW_CELL_SUFFIX)) {
+ return Arrays.copyOfRange(qualifier,
+ qualOffset,
+ qualOffset + (qualLength - SHADOW_CELL_SUFFIX.length));
+ }
+
+ throw new IllegalArgumentException(
+ "Can't find shadow cell suffix in qualifier "
+ + Bytes.toString(qualifier));
+ }
+
+ /**
+ * Returns the qualifier length removing the shadow cell suffix. In case that que suffix is not found,
+ * just returns the length of the qualifier passed.
+ * @param qualifier
+ * the qualifier to remove the suffix from
+ * @param qualOffset
+ * the offset where the qualifier starts
+ * @param qualLength
+ * the qualifier length
+ * @return the qualifier length without the suffix
+ */
+ public static int qualifierLengthFromShadowCellQualifier(byte[] qualifier, int qualOffset, int qualLength) {
+
+ if (endsWith(qualifier, qualOffset, qualLength, SHADOW_CELL_SUFFIX)) {
+ return qualLength - SHADOW_CELL_SUFFIX.length;
+ }
+
+ return qualLength;
+
+ }
+
+ /**
+ * Complement to matchingQualifier() methods in HBase's CellUtil.class
+ * @param left
+ * the cell to compare the qualifier
+ * @param qualArray
+ * the explicit qualifier array passed
+ * @param qualOffset
+ * the explicit qualifier offset passed
+ * @param qualLen
+ * the explicit qualifier length passed
+ * @return whether the qualifiers are equal or not
+ */
+ public static boolean matchingQualifier(final Cell left, final byte[] qualArray, int qualOffset, int qualLen) {
+ return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), left.getQualifierLength(),
+ qualArray, qualOffset, qualLen);
+ }
+
+ /**
+ * Check that the cell passed meets the requirements for a valid cell identifier with Omid. Basically, users can't:
+ * 1) specify a timestamp
+ * 2) use a particular suffix in the qualifier
+ */
+ public static void validateCell(Cell cell, long startTimestamp) {
+ // Throw exception if timestamp is set by the user
+ if (cell.getTimestamp() != HConstants.LATEST_TIMESTAMP
+ && cell.getTimestamp() != startTimestamp) {
+ throw new IllegalArgumentException(
+ "Timestamp not allowed in transactional user operations");
+ }
+ // Throw exception if using a non-allowed qualifier
+ if (isShadowCell(cell)) {
+ throw new IllegalArgumentException(
+ "Reserved string used in column qualifier");
+ }
+ }
+
+ /**
+ * Returns whether a cell contains a qualifier that is a shadow cell
+ * column qualifier or not.
+ * @param cell
+ * the cell to check if contains the shadow cell qualifier
+ * @return whether the cell passed contains a shadow cell qualifier or not
+ */
+ public static boolean isShadowCell(Cell cell) {
+ byte[] qualifier = cell.getQualifierArray();
+ int qualOffset = cell.getQualifierOffset();
+ int qualLength = cell.getQualifierLength();
+
+ return endsWith(qualifier, qualOffset, qualLength, SHADOW_CELL_SUFFIX);
+ }
+
+ private static boolean endsWith(byte[] value, int offset, int length, byte[] suffix) {
+ if (length <= suffix.length) {
+ return false;
+ }
+
+ int suffixOffset = offset + length - suffix.length;
+ int result = Bytes.compareTo(value, suffixOffset, suffix.length,
+ suffix, 0, suffix.length);
+ return result == 0;
+ }
+
+ /**
+ * Returns if a cell is marked as a tombstone.
+ * @param cell
+ * the cell to check
+ * @return whether the cell is marked as a tombstone or not
+ */
+ public static boolean isTombstone(Cell cell) {
+ return CellUtil.matchingValue(cell, DELETE_TOMBSTONE);
+ }
+
+ /**
+ * Returns a new shadow cell created from a particular cell.
+ * @param cell
+ * the cell to reconstruct the shadow cell from.
+ * @param shadowCellValue
+ * the value for the new shadow cell created
+ * @return the brand-new shadow cell
+ */
+ public static Cell buildShadowCellFromCell(Cell cell, byte[] shadowCellValue) {
+ byte[] shadowCellQualifier = addShadowCellSuffix(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ return new KeyValue(
+ cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ shadowCellQualifier, 0, shadowCellQualifier.length,
+ cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()),
+ shadowCellValue, 0, shadowCellValue.length);
+ }
+
+ /**
+ * Analyzes a list of cells, associating the corresponding shadow cell if present.
+ *
+ * @param cells
+ * the list of cells to classify
+ * @return a sorted map associating each cell with its shadow cell
+ */
+ public static SortedMap<Cell, Optional<Cell>> mapCellsToShadowCells(List<Cell> cells) {
+
+ SortedMap<Cell, Optional<Cell>> cellToShadowCellMap
+ = new TreeMap<Cell, Optional<Cell>>(new CellComparator());
+
+ Map<CellId, Cell> cellIdToCellMap = new HashMap<CellId, Cell>();
+ for (Cell cell : cells) {
+ if (!isShadowCell(cell)) {
+ CellId key = new CellId(cell, false);
+ if (cellIdToCellMap.containsKey(key)) {
+ // Get the current cell and compare the values
+ Cell storedCell = cellIdToCellMap.get(key);
+ if (CellUtil.matchingValue(cell, storedCell)) {
+ // TODO: Should we check also here the MVCC and swap if its greater???
+ // Values are the same, ignore
+ } else {
+ if (cell.getMvccVersion() > storedCell.getMvccVersion()) { // Swap values
+ Optional<Cell> previousValue = cellToShadowCellMap.remove(storedCell);
+ Preconditions.checkNotNull(previousValue, "Should contain an Optional<Cell> value");
+ cellIdToCellMap.put(key, cell);
+ cellToShadowCellMap.put(cell, previousValue);
+ } else {
+ LOG.warn("Cell {} with an earlier MVCC found. Ignoring...", cell);
+ }
+ }
+ } else {
+ cellIdToCellMap.put(key, cell);
+ cellToShadowCellMap.put(cell, Optional.<Cell>absent());
+ }
+ } else {
+ CellId key = new CellId(cell, true);
+ if (cellIdToCellMap.containsKey(key)) {
+ Cell originalCell = cellIdToCellMap.get(key);
+ cellToShadowCellMap.put(originalCell, Optional.of(cell));
+ } else {
+ LOG.trace("Map does not contain key {}", key);
+ }
+ }
+ }
+
+ return cellToShadowCellMap;
+ }
+
+ private static class CellId {
+
+ private static final int MIN_BITS = 32;
+
+ private final Cell cell;
+ private final boolean isShadowCell;
+
+ CellId(Cell cell, boolean isShadowCell) {
+
+ this.cell = cell;
+ this.isShadowCell = isShadowCell;
+
+ }
+
+ Cell getCell() {
+ return cell;
+ }
+
+ boolean isShadowCell() {
+ return isShadowCell;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this)
+ return true;
+ if (!(o instanceof CellId))
+ return false;
+ CellId otherCellId = (CellId) o;
+ Cell otherCell = otherCellId.getCell();
+
+ // Row comparison
+ if (!CellUtil.matchingRow(otherCell, cell)) {
+ return false;
+ }
+
+ // Family comparison
+ if (!CellUtil.matchingFamily(otherCell, cell)) {
+ return false;
+ }
+
+ // Qualifier comparison
+ if (isShadowCell()) {
+ int qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ if (!matchingQualifier(otherCell,
+ cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength)) {
+ return false;
+ }
+ } else {
+ if (!CellUtil.matchingQualifier(otherCell, cell)) {
+ return false;
+ }
+ }
+
+ // Timestamp comparison
+ return otherCell.getTimestamp() == cell.getTimestamp();
+
+ }
+
+ @Override
+ public int hashCode() {
+ Hasher hasher = Hashing.goodFastHash(MIN_BITS).newHasher();
+ hasher.putBytes(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
+ hasher.putBytes(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
+ int qualifierLength = cell.getQualifierLength();
+ if (isShadowCell()) { // Update qualifier length when qualifier is shadow cell
+ qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ }
+ hasher.putBytes(cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength);
+ hasher.putLong(cell.getTimestamp());
+ return hasher.hash().asInt();
+ }
+
+ @Override
+ public String toString() {
+ ToStringHelper helper = Objects.toStringHelper(this);
+ helper.add("row", Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
+ helper.add("family", Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()));
+ helper.add("is shadow cell?", isShadowCell);
+ helper.add("qualifier",
+ Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()));
+ if (isShadowCell()) {
+ int qualifierLength = qualifierLengthFromShadowCellQualifier(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ helper.add("qualifier whithout shadow cell suffix",
+ Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength));
+ }
+ helper.add("ts", cell.getTimestamp());
+ return helper.toString();
+ }
+ }
+
+}
[15/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestTimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestTimestampOracle.java b/tso-server/src/test/java/org/apache/omid/tso/TestTimestampOracle.java
new file mode 100644
index 0000000..66a70fb
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestTimestampOracle.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class TestTimestampOracle {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTimestampOracle.class);
+
+ @Mock
+ private MetricsRegistry metrics;
+ @Mock
+ private Panicker panicker;
+ @Mock
+ private TimestampStorage timestampStorage;
+
+ // Component under test
+ @InjectMocks
+ private TimestampOracleImpl timestampOracle;
+
+ @BeforeMethod(alwaysRun = true, timeOut = 30_000)
+ public void initMocksAndComponents() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @Test(timeOut = 10_000)
+ public void testMonotonicTimestampGrowth() throws Exception {
+
+ // Intialize component under test
+ timestampOracle.initialize();
+
+ long last = timestampOracle.next();
+ for (int i = 0; i < (3 * TimestampOracleImpl.TIMESTAMP_BATCH); i++) {
+ long current = timestampOracle.next();
+ assertEquals(current, last + 1, "Not monotonic growth");
+ last = current;
+ }
+ assertTrue(timestampOracle.getLast() == last);
+ LOG.info("Last timestamp: {}", last);
+ }
+
+ @Test(timeOut = 10_000)
+ public void testTimestampOraclePanicsWhenTheStorageHasProblems() throws Exception {
+
+ // Intialize component under test
+ timestampOracle.initialize();
+
+ // Cause an exception when updating the max timestamp
+ final CountDownLatch updateMaxTimestampMethodCalled = new CountDownLatch(1);
+ doAnswer(new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
+ updateMaxTimestampMethodCalled.countDown();
+ throw new RuntimeException("Out of memory or something");
+ }
+ }).when(timestampStorage).updateMaxTimestamp(anyLong(), anyLong());
+
+ // Make the previous exception to be thrown
+ Thread allocThread = new Thread("AllocThread") {
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ timestampOracle.next();
+ }
+ } catch (IOException ioe) {
+ LOG.error("Shouldn't occur");
+ }
+ }
+ };
+ allocThread.start();
+
+ updateMaxTimestampMethodCalled.await();
+
+ // Verify that it has blown up
+ verify(panicker, atLeastOnce()).panic(anyString(), any(Throwable.class));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientAccessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientAccessor.java b/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientAccessor.java
new file mode 100644
index 0000000..74ed196
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientAccessor.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import org.apache.statemachine.StateMachine.FsmImpl;
+
+public class TSOClientAccessor {
+
+ public static void closeChannel(TSOClient tsoClient) throws InterruptedException {
+ FsmImpl fsm = (FsmImpl) tsoClient.fsm;
+ TSOClient.ConnectedState connectedState = (TSOClient.ConnectedState) fsm.getState();
+ connectedState.channel.close().await();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientOneShot.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientOneShot.java b/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientOneShot.java
new file mode 100644
index 0000000..ff60753
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientOneShot.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import org.apache.omid.proto.TSOProto;
+import org.apache.omid.proto.TSOProto.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Communication endpoint for TSO clients.
+ */
+public class TSOClientOneShot {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOClientOneShot.class);
+
+ private final String host;
+ private final int port;
+
+ public TSOClientOneShot(String host, int port) {
+
+ this.host = host;
+ this.port = port;
+
+ }
+
+ public TSOProto.Response makeRequest(TSOProto.Request request)
+ throws InterruptedException, ExecutionException {
+ TSOClientRaw raw = new TSOClientRaw(host, port);
+
+ // do handshake
+ TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
+ handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
+ raw.write(TSOProto.Request.newBuilder()
+ .setHandshakeRequest(handshake.build()).build());
+ Response response = raw.getResponse().get();
+ assert (response.getHandshakeResponse().getClientCompatible());
+
+ raw.write(request);
+ response = raw.getResponse().get();
+
+ raw.close();
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientRaw.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientRaw.java b/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientRaw.java
new file mode 100644
index 0000000..beb6c47
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TSOClientRaw.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.proto.TSOProto;
+import org.apache.omid.proto.TSOProto.Response;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Raw client for communicating with tso server directly with protobuf messages
+ */
+public class TSOClientRaw {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOClientRaw.class);
+
+ private final BlockingQueue<SettableFuture<Response>> responseQueue
+ = new ArrayBlockingQueue<SettableFuture<Response>>(5);
+ private final Channel channel;
+
+ public TSOClientRaw(String host, int port) throws InterruptedException, ExecutionException {
+ // Start client with Nb of active threads = 3 as maximum.
+ ChannelFactory factory = new NioClientSocketChannelFactory(
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), 3);
+ // Create the bootstrap
+ ClientBootstrap bootstrap = new ClientBootstrap(factory);
+
+ InetSocketAddress addr = new InetSocketAddress(host, port);
+
+ ChannelPipeline pipeline = bootstrap.getPipeline();
+ pipeline.addLast("lengthbaseddecoder",
+ new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
+ pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+ pipeline.addLast("protobufdecoder",
+ new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
+ pipeline.addLast("protobufencoder", new ProtobufEncoder());
+
+ Handler handler = new Handler();
+ pipeline.addLast("handler", handler);
+
+ bootstrap.setOption("tcpNoDelay", true);
+ bootstrap.setOption("keepAlive", true);
+ bootstrap.setOption("reuseAddress", true);
+ bootstrap.setOption("connectTimeoutMillis", 100);
+
+ ChannelFuture channelFuture = bootstrap.connect(addr).await();
+ channel = channelFuture.getChannel();
+ }
+
+ public void write(TSOProto.Request request) {
+ channel.write(request);
+ }
+
+ public Future<Response> getResponse() throws InterruptedException {
+ SettableFuture<Response> future = SettableFuture.<Response>create();
+ responseQueue.put(future);
+ return future;
+ }
+
+ public void close() throws InterruptedException {
+ responseQueue.put(SettableFuture.<Response>create());
+ channel.close();
+ }
+
+ private class Handler extends SimpleChannelHandler {
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
+ LOG.info("Message received", e);
+ if (e.getMessage() instanceof Response) {
+ Response resp = (Response) e.getMessage();
+ try {
+ SettableFuture<Response> future = responseQueue.take();
+ future.set(resp);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted in handler", ie);
+ }
+ } else {
+ LOG.warn("Received unknown message", e.getMessage());
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
+ LOG.info("Exception received", e.getCause());
+ try {
+ SettableFuture<Response> future = responseQueue.take();
+ future.setException(e.getCause());
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted handling exception", ie);
+ }
+ }
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e)
+ throws Exception {
+ LOG.info("Disconnected");
+ try {
+ SettableFuture<Response> future = responseQueue.take();
+ future.setException(new ConnectionException());
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted handling exception", ie);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java b/tso-server/src/test/java/org/apache/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
new file mode 100644
index 0000000..9f7263e
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java
@@ -0,0 +1,233 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.collect.Sets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import org.apache.omid.TestUtils;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.tso.TSOMockModule;
+import org.apache.omid.tso.TSOServer;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.omid.tso.util.DummyCellIdImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertNotNull;
+
+public class TestIntegrationOfTSOClientServerBasicFunctionality {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestIntegrationOfTSOClientServerBasicFunctionality.class);
+
+ private static final String TSO_SERVER_HOST = "localhost";
+ private int tsoServerPortForTest;
+
+ // Cells for tests
+ private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
+ private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
+
+ // Required infrastructure for TSO tsoClient-server integration testing
+ private TSOServer tsoServer;
+ private TSOClient tsoClient;
+ private TSOClient justAnotherTSOClient;
+ private CommitTable.Client commitTableClient;
+
+ @BeforeClass
+ public void setup() throws Exception {
+
+ tsoServerPortForTest = TestUtils.getFreeLocalPort();
+
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setMaxItems(1000);
+ tsoConfig.setPort(tsoServerPortForTest);
+ Module tsoServerMockModule = new TSOMockModule(tsoConfig);
+ Injector injector = Guice.createInjector(tsoServerMockModule);
+
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ commitTableClient = commitTable.getClient();
+
+ LOG.info("==================================================================================================");
+ LOG.info("======================================= Init TSO Server ==========================================");
+ LOG.info("==================================================================================================");
+
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_SERVER_HOST, tsoServerPortForTest, 100);
+
+ LOG.info("==================================================================================================");
+ LOG.info("===================================== TSO Server Initialized =====================================");
+ LOG.info("==================================================================================================");
+
+ LOG.info("==================================================================================================");
+ LOG.info("======================================= Setup TSO Clients ========================================");
+ LOG.info("==================================================================================================");
+
+ // Configure direct connection to the server
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + tsoServerPortForTest);
+
+ tsoClient = TSOClient.newInstance(tsoClientConf);
+ justAnotherTSOClient = TSOClient.newInstance(tsoClientConf);
+
+ LOG.info("==================================================================================================");
+ LOG.info("===================================== TSO Clients Initialized ====================================");
+ LOG.info("==================================================================================================");
+
+ Thread.currentThread().setName("Test Thread");
+
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+
+ tsoClient.close().get();
+
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, tsoServerPortForTest, 1000);
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTimestampsOrderingGrowMonotonically() throws Exception {
+ long referenceTimestamp;
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ referenceTimestamp = startTsTx1;
+
+ long startTsTx2 = tsoClient.getNewStartTimestamp().get();
+ assertEquals(startTsTx2, ++referenceTimestamp, "Should grow monotonically");
+ assertTrue(startTsTx2 > startTsTx1, "Two timestamps obtained consecutively should grow");
+
+ long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1)).get();
+ assertEquals(commitTsTx2, ++referenceTimestamp, "Should grow monotonically");
+
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c2)).get();
+ assertEquals(commitTsTx1, ++referenceTimestamp, "Should grow monotonically");
+
+ long startTsTx3 = tsoClient.getNewStartTimestamp().get();
+ assertEquals(startTsTx3, ++referenceTimestamp, "Should grow monotonically");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSimpleTransactionWithNoWriteSetCanCommit() throws Exception {
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.<CellId>newHashSet()).get();
+ assertTrue(commitTsTx1 > startTsTx1);
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTransactionWithMassiveWriteSetCanCommit() throws Exception {
+ long startTs = tsoClient.getNewStartTimestamp().get();
+
+ Set<CellId> cells = new HashSet<>();
+ for (int i = 0; i < 1_000_000; i++) {
+ cells.add(new DummyCellIdImpl(i));
+ }
+
+ long commitTs = tsoClient.commit(startTs, cells).get();
+ assertTrue(commitTs > startTs, "Commit TS should be higher than Start TS");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testMultipleSerialCommitsDoNotConflict() throws Exception {
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
+ assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be greater than Start TS");
+
+ long startTsTx2 = tsoClient.getNewStartTimestamp().get();
+ assertTrue(startTsTx2 > commitTsTx1, "TS should grow monotonically");
+
+ long commitTsTx2 = tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
+ assertTrue(commitTsTx2 > startTsTx2, "Commit TS must be greater than Start TS");
+
+ long startTsTx3 = tsoClient.getNewStartTimestamp().get();
+ long commitTsTx3 = tsoClient.commit(startTsTx3, Sets.newHashSet(c2)).get();
+ assertTrue(commitTsTx3 > startTsTx3, "Commit TS must be greater than Start TS");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitWritesToCommitTable() throws Exception {
+ long startTsForTx1 = tsoClient.getNewStartTimestamp().get();
+ long startTsForTx2 = tsoClient.getNewStartTimestamp().get();
+ assertTrue(startTsForTx2 > startTsForTx1, "Start TS should grow");
+
+ assertFalse(commitTableClient.getCommitTimestamp(startTsForTx1).get().isPresent(),
+ "Commit TS for Tx1 shouldn't appear in Commit Table");
+
+ long commitTsForTx1 = tsoClient.commit(startTsForTx1, Sets.newHashSet(c1)).get();
+ assertTrue(commitTsForTx1 > startTsForTx1, "Commit TS should be higher than Start TS for the same tx");
+
+ Long commitTs1InCommitTable = commitTableClient.getCommitTimestamp(startTsForTx1).get().get().getValue();
+ assertNotNull("Tx is committed, should return as such from Commit Table", commitTs1InCommitTable);
+ assertEquals(commitTsForTx1, (long) commitTs1InCommitTable,
+ "getCommitTimestamp() & commit() should report same Commit TS value for same tx");
+ assertTrue(commitTs1InCommitTable > startTsForTx2, "Commit TS should be higher than tx's Start TS");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTwoConcurrentTxWithOverlappingWritesetsHaveConflicts() throws Exception {
+ long startTsTx1 = tsoClient.getNewStartTimestamp().get();
+ long startTsTx2 = tsoClient.getNewStartTimestamp().get();
+ assertTrue(startTsTx2 > startTsTx1, "Second TX should have higher TS");
+
+ long commitTsTx1 = tsoClient.commit(startTsTx1, Sets.newHashSet(c1)).get();
+ assertTrue(commitTsTx1 > startTsTx1, "Commit TS must be higher than Start TS for the same tx");
+
+ try {
+ tsoClient.commit(startTsTx2, Sets.newHashSet(c1, c2)).get();
+ Assert.fail("Second TX should fail on commit");
+ } catch (ExecutionException ee) {
+ assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void testConflictsAndMonotonicallyTimestampGrowthWithTwoDifferentTSOClients() throws Exception {
+ long startTsTx1Client1 = tsoClient.getNewStartTimestamp().get();
+ long startTsTx2Client1 = tsoClient.getNewStartTimestamp().get();
+ long startTsTx3Client1 = tsoClient.getNewStartTimestamp().get();
+
+ tsoClient.commit(startTsTx1Client1, Sets.newHashSet(c1)).get();
+ try {
+ tsoClient.commit(startTsTx3Client1, Sets.newHashSet(c1, c2)).get();
+ Assert.fail("Second commit should fail as conflicts with the previous concurrent one");
+ } catch (ExecutionException ee) {
+ assertEquals(AbortException.class, ee.getCause().getClass(), "Should have aborted");
+ }
+ long startTsTx4Client2 = justAnotherTSOClient.getNewStartTimestamp().get();
+
+ assertFalse(commitTableClient.getCommitTimestamp(startTsTx3Client1).get().isPresent(), "Tx3 didn't commit");
+ long commitTSTx1 = commitTableClient.getCommitTimestamp(startTsTx1Client1).get().get().getValue();
+ assertTrue(commitTSTx1 > startTsTx2Client1, "Tx1 committed after Tx2 started");
+ assertTrue(commitTSTx1 < startTsTx4Client2, "Tx1 committed before Tx4 started on the other TSO Client");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientConnectionToTSO.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientConnectionToTSO.java b/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientConnectionToTSO.java
new file mode 100644
index 0000000..2b4c312
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientConnectionToTSO.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.omid.TestUtils;
+import org.apache.omid.tso.HALeaseManagementModule;
+import org.apache.omid.tso.TSOMockModule;
+import org.apache.omid.tso.TSOServer;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.omid.tso.VoidLeaseManagementModule;
+import org.apache.statemachine.StateMachine.FsmImpl;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.utils.CloseableUtils;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class TestTSOClientConnectionToTSO {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientConnectionToTSO.class);
+
+ // Constants and variables for component connectivity
+ private static final String TSO_HOST = "localhost";
+ private static final String CURRENT_TSO_PATH = "/current_tso_path";
+ private static final String TSO_LEASE_PATH = "/tso_lease_path";
+
+ private int tsoPortForTest;
+ private String zkClusterForTest;
+
+ private Injector injector = null;
+
+ private TestingServer zkServer;
+
+ private CuratorFramework zkClient;
+ private TSOServer tsoServer;
+
+ @BeforeMethod
+ public void beforeMethod() throws Exception {
+
+ tsoPortForTest = TestUtils.getFreeLocalPort();
+
+ int zkPortForTest = TestUtils.getFreeLocalPort();
+ zkClusterForTest = TSO_HOST + ":" + zkPortForTest;
+ LOG.info("Starting ZK Server in port {}", zkPortForTest);
+ zkServer = TestUtils.provideTestingZKServer(zkPortForTest);
+ LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
+
+ zkClient = TestUtils.provideConnectedZKClient(zkClusterForTest);
+
+ Stat stat;
+ try {
+ zkClient.delete().forPath(CURRENT_TSO_PATH);
+ stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
+ assertNull(stat, CURRENT_TSO_PATH + " should not exist");
+ } catch (NoNodeException e) {
+ LOG.info("{} ZNode did not exist", CURRENT_TSO_PATH);
+ }
+
+ }
+
+ @AfterMethod
+ public void afterMethod() {
+
+ zkClient.close();
+
+ CloseableUtils.closeQuietly(zkServer);
+ zkServer = null;
+ LOG.info("ZK Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testUnsuccessfulConnectionToTSO() throws Exception {
+
+ // When no HA node for TSOServer is found & no host:port config exists
+ // we should get an exception when getting the client
+ try {
+ TSOClient.newInstance(new OmidClientConfiguration());
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSuccessfulConnectionToTSOWithHostAndPort() throws Exception {
+
+ // Launch a TSO WITHOUT publishing the address in HA...
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setMaxItems(1000);
+ tsoConfig.setPort(tsoPortForTest);
+ tsoConfig.setLeaseModule(new VoidLeaseManagementModule());
+ injector = Guice.createInjector(new TSOMockModule(tsoConfig));
+ LOG.info("Starting TSO");
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading TSO");
+
+ // When no HA node for TSOServer is found we should get a connection
+ // to the TSO through the host:port configured...
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString("localhost:" + tsoPortForTest);
+ tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // ... so we should get responses from the methods
+ Long startTS = tsoClient.getNewStartTimestamp().get();
+ LOG.info("Start TS {} ", startTS);
+ assertEquals(startTS.longValue(), 1);
+
+ // Close the tsoClient connection and stop the TSO Server
+ tsoClient.close().get();
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("TSO Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSuccessfulConnectionToTSOThroughZK() throws Exception {
+
+ // Launch a TSO publishing the address in HA...
+ TSOServerConfig config = new TSOServerConfig();
+ config.setMaxItems(1000);
+ config.setPort(tsoPortForTest);
+ config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
+ injector = Guice.createInjector(new TSOMockModule(config));
+ LOG.info("Starting TSO");
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading TSO");
+
+ waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
+
+ // When a HA node for TSOServer is found we should get a connection
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionType(OmidClientConfiguration.ConnType.HA);
+ tsoClientConf.setConnectionString(zkClusterForTest);
+ tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // ... so we should get responses from the methods
+ Long startTS = tsoClient.getNewStartTimestamp().get();
+ LOG.info("Start TS {} ", startTS);
+ assertEquals(startTS.longValue(), 1);
+
+ // Close the tsoClient connection and stop the TSO Server
+ tsoClient.close().get();
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("TSO Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testSuccessOfTSOClientReconnectionsToARestartedTSOWithZKPublishing() throws Exception {
+
+ // Start a TSO with HA...
+ TSOServerConfig config = new TSOServerConfig();
+ config.setMaxItems(1000);
+ config.setPort(tsoPortForTest);
+ config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
+ injector = Guice.createInjector(new TSOMockModule(config));
+ LOG.info("Starting Initial TSO");
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading TSO");
+
+ waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
+
+ // Then create the TSO Client under test...
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionType(OmidClientConfiguration.ConnType.HA);
+ tsoClientConf.setConnectionString(zkClusterForTest);
+ tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // ... and check that initially we get responses from the methods
+ Long startTS = tsoClient.getNewStartTimestamp().get();
+ LOG.info("Start TS {} ", startTS);
+ assertEquals(startTS.longValue(), 1);
+
+ // Then stop the server...
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("Initial TSO Server Stopped");
+
+ Thread.sleep(1500); // ...allow the client to receive disconnection event...
+ // ... and check that we get a conn exception when trying to access the client
+ try {
+ startTS = tsoClient.getNewStartTimestamp().get();
+ fail();
+ } catch (ExecutionException e) {
+ LOG.info("Exception expected");
+ // Internal accessor to fsm to do the required checkings
+ FsmImpl fsm = (FsmImpl) tsoClient.fsm;
+ assertEquals(e.getCause().getClass(), ConnectionException.class);
+ assertTrue(fsm.getState().getClass().equals(TSOClient.ConnectionFailedState.class)
+ ||
+ fsm.getState().getClass().equals(TSOClient.DisconnectedState.class));
+ }
+
+ // After that, simulate that a new TSO has been launched...
+ Injector newInjector = Guice.createInjector(new TSOMockModule(config));
+ LOG.info("Re-Starting again the TSO");
+ tsoServer = newInjector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
+ LOG.info("Finished loading restarted TSO");
+
+ // Finally re-check that, eventually, we can get a new value from the new TSO...
+ boolean reconnectionActive = false;
+ while (!reconnectionActive) {
+ try {
+ startTS = tsoClient.getNewStartTimestamp().get();
+ reconnectionActive = true;
+ } catch (ExecutionException e) {
+ // Expected
+ }
+ }
+ assertNotNull(startTS);
+
+ // ...and stop the server
+ tsoServer.stopAndWait();
+ TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
+ LOG.info("Restarted TSO Server Stopped");
+ }
+
+ private void waitTillTsoRegisters(CuratorFramework zkClient) throws Exception {
+ while (true) {
+ try {
+ Stat stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
+ if (stat == null) {
+ continue;
+ }
+ LOG.info("TSO registered in HA with path {}={}", CURRENT_TSO_PATH, stat.toString());
+ if (stat.toString().length() == 0) {
+ continue;
+ }
+ return;
+ } catch (Exception e) {
+ LOG.debug("TSO still has not registered yet, sleeping...", e);
+ Thread.sleep(500);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java b/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
new file mode 100644
index 0000000..44c4858
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java
@@ -0,0 +1,423 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.collect.Sets;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import org.apache.omid.TestUtils;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.proto.TSOProto;
+import org.apache.omid.tso.PausableTimestampOracle;
+import org.apache.omid.tso.TSOMockModule;
+import org.apache.omid.tso.TSOServer;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.omid.tso.TimestampOracle;
+import org.apache.omid.tso.util.DummyCellIdImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+public class TestTSOClientRequestAndResponseBehaviours {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientRequestAndResponseBehaviours.class);
+
+ private static final String TSO_SERVER_HOST = "localhost";
+ private static final int TSO_SERVER_PORT = 1234;
+
+ private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
+ private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
+
+ private final static Set<CellId> testWriteSet = Sets.newHashSet(c1, c2);
+
+ private OmidClientConfiguration tsoClientConf;
+
+ // Required infrastructure for TSOClient test
+ private TSOServer tsoServer;
+ private PausableTimestampOracle pausableTSOracle;
+ private CommitTable commitTable;
+
+ @BeforeClass
+ public void setup() throws Exception {
+
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setMaxItems(1000);
+ tsoConfig.setPort(TSO_SERVER_PORT);
+ Module tsoServerMockModule = new TSOMockModule(tsoConfig);
+ Injector injector = Guice.createInjector(tsoServerMockModule);
+
+ LOG.info("==================================================================================================");
+ LOG.info("======================================= Init TSO Server ==========================================");
+ LOG.info("==================================================================================================");
+
+ tsoServer = injector.getInstance(TSOServer.class);
+ tsoServer.startAndWait();
+ TestUtils.waitForSocketListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 100);
+
+ LOG.info("==================================================================================================");
+ LOG.info("===================================== TSO Server Initialized =====================================");
+ LOG.info("==================================================================================================");
+
+ pausableTSOracle = (PausableTimestampOracle) injector.getInstance(TimestampOracle.class);
+ commitTable = injector.getInstance(CommitTable.class);
+
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+
+ tsoServer.stopAndWait();
+ tsoServer = null;
+ TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 1000);
+
+ }
+
+ @BeforeMethod
+ public void beforeMethod() {
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+
+ this.tsoClientConf = tsoClientConf;
+
+ }
+
+ @AfterMethod
+ public void afterMethod() {
+
+ pausableTSOracle.resume();
+
+ }
+
+ /**
+ * Test to ensure TSOClient timeouts are cancelled.
+ * At some point a bug was detected because the TSOClient timeouts were not cancelled, and as timestamp requests
+ * had no way to be correlated to timestamp responses, random requests were just timed out after a certain time.
+ * We send a lot of timestamp requests, and wait for them to complete.
+ * Ensure that the next request doesn't get hit by the timeouts of the previous
+ * requests. (i.e. make sure we cancel timeouts)
+ */
+ @Test(timeOut = 30_000)
+ public void testTimeoutsAreCancelled() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ int requestTimeoutInMs = 500;
+ int requestMaxRetries = 5;
+ LOG.info("Request timeout {} ms; Max retries {}", requestTimeoutInMs, requestMaxRetries);
+ Future<Long> f = null;
+ for (int i = 0; i < (requestMaxRetries * 10); i++) {
+ f = client.getNewStartTimestamp();
+ }
+ if (f != null) {
+ f.get();
+ }
+ pausableTSOracle.pause();
+ long msToSleep = ((long) (requestTimeoutInMs * 0.75));
+ LOG.info("Sleeping for {} ms", msToSleep);
+ TimeUnit.MILLISECONDS.sleep(msToSleep);
+ f = client.getNewStartTimestamp();
+ msToSleep = ((long) (requestTimeoutInMs * 0.9));
+ LOG.info("Sleeping for {} ms", msToSleep);
+ TimeUnit.MILLISECONDS.sleep(msToSleep);
+ LOG.info("Resuming");
+ pausableTSOracle.resume();
+ f.get();
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitGetsServiceUnavailableExceptionWhenCommunicationFails() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestMaxRetries(0);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ List<Long> startTimestamps = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ startTimestamps.add(client.getNewStartTimestamp().get());
+ }
+
+ pausableTSOracle.pause();
+
+ List<Future<Long>> futures = new ArrayList<>();
+ for (long s : startTimestamps) {
+ futures.add(client.commit(s, Sets.<CellId>newHashSet()));
+ }
+ TSOClientAccessor.closeChannel(client);
+
+ for (Future<Long> f : futures) {
+ try {
+ f.get();
+ fail("Shouldn't be able to complete");
+ } catch (ExecutionException ee) {
+ assertTrue(ee.getCause() instanceof ServiceUnavailableException,
+ "Should be a service unavailable exception");
+ }
+ }
+ }
+
+ /**
+ * Test that if a client tries to make a request without handshaking, it will be disconnected.
+ */
+ @Test(timeOut = 30_000)
+ public void testHandshakeBetweenOldClientAndCurrentServer() throws Exception {
+
+ TSOClientRaw raw = new TSOClientRaw(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ TSOProto.Request request = TSOProto.Request.newBuilder()
+ .setTimestampRequest(TSOProto.TimestampRequest.newBuilder().build())
+ .build();
+ raw.write(request);
+ try {
+ raw.getResponse().get();
+ fail("Channel should be closed");
+ } catch (ExecutionException ee) {
+ assertEquals(ee.getCause().getClass(), ConnectionException.class, "Should be channel closed exception");
+ }
+ raw.close();
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Test duplicate commits
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * This tests the case where messages arrive at the TSO out of order. This can happen in the case
+ * the channel get dropped and the retry is done in a new channel. However, the TSO will respond with
+ * aborted to the original message because the retry was already committed and it would be prohibitively
+ * expensive to check all non-retry requests to see if they are already committed. For this reason
+ * a client must ensure that if it is sending a retry due to a socket error, the previous channel
+ * must be entirely closed so that it will not actually receive the abort response. TCP guarantees
+ * that this doesn't happen in non-socket error cases.
+ *
+ */
+ @Test(timeOut = 30_000)
+ public void testOutOfOrderMessages() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long ts1 = client.getNewStartTimestamp().get();
+
+ TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
+ TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
+ assertFalse(response1.getCommitResponse().getAborted(), "Retry Transaction should commit");
+ assertTrue(response2.getCommitResponse().getAborted(), "Transaction should abort");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testDuplicateCommitAborting() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ long ts2 = client.getNewStartTimestamp().get();
+ client.commit(ts2, testWriteSet).get();
+
+ TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
+ TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
+ assertTrue(response1.getCommitResponse().getAborted(), "Transaction should abort");
+ assertTrue(response2.getCommitResponse().getAborted(), "Retry commit should abort");
+ }
+
+ @Test(timeOut = 30_000)
+ public void testDuplicateCommit() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long ts1 = client.getNewStartTimestamp().get();
+
+ TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
+ TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
+ assertEquals(response2.getCommitResponse().getCommitTimestamp(),
+ response1.getCommitResponse().getCommitTimestamp(),
+ "Commit timestamp should be the same");
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Test TSOClient retry behaviour
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Test(timeOut = 30_000)
+ public void testCommitCanSucceedWhenChannelDisconnected() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ pausableTSOracle.pause();
+ TSOFuture<Long> future = client.commit(ts1, testWriteSet);
+ TSOClientAccessor.closeChannel(client);
+ pausableTSOracle.resume();
+ future.get();
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitCanSucceedWithMultipleTimeouts() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestTimeoutInMs(100);
+ testTSOClientConf.setRequestMaxRetries(10000);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ pausableTSOracle.pause();
+ TSOFuture<Long> future = client.commit(ts1, testWriteSet);
+ TimeUnit.SECONDS.sleep(1);
+ pausableTSOracle.resume();
+ future.get();
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitFailWhenTSOIsDown() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestTimeoutInMs(100);
+ testTSOClientConf.setRequestMaxRetries(10);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ long ts1 = client.getNewStartTimestamp().get();
+ pausableTSOracle.pause();
+ TSOFuture<Long> future = client.commit(ts1, testWriteSet);
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ assertEquals(e.getCause().getClass(), ServiceUnavailableException.class,
+ "Should be a ServiceUnavailableExeption");
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTimestampRequestSucceedWithMultipleTimeouts() throws Exception {
+
+ OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
+ testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
+ testTSOClientConf.setRequestTimeoutInMs(100);
+ testTSOClientConf.setRequestMaxRetries(10000);
+ TSOClient client = TSOClient.newInstance(testTSOClientConf);
+
+ pausableTSOracle.pause();
+ Future<Long> future = client.getNewStartTimestamp();
+ TimeUnit.SECONDS.sleep(1);
+ pausableTSOracle.resume();
+ future.get();
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // The next 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
+ // (They exercise the communication protocol) TODO Remove???
+ // ----------------------------------------------------------------------------------------------------------------
+ @Test
+ public void testCommitTimestampPresentInCommitTableReturnsCommit() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long tx1ST = client.getNewStartTimestamp().get();
+
+ clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ assertFalse(response.getCommitResponse().getAborted(), "Transaction should be committed");
+ assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
+ assertEquals(response.getCommitResponse().getCommitTimestamp(), tx1ST + 1);
+ }
+
+ @Test
+ public void testInvalidCommitTimestampPresentInCommitTableReturnsAbort() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long tx1ST = client.getNewStartTimestamp().get();
+ // Invalidate the transaction
+ commitTable.getClient().tryInvalidateTransaction(tx1ST);
+
+ clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ assertTrue(response.getCommitResponse().getAborted(), "Transaction should be aborted");
+ assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
+ assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
+ }
+
+ @Test
+ public void testCommitTimestampNotPresentInCommitTableReturnsAnAbort() throws Exception {
+
+ TSOClient client = TSOClient.newInstance(tsoClientConf);
+ TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
+
+ long tx1ST = client.getNewStartTimestamp().get();
+
+ clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+
+ // Simulate remove entry from the commit table before exercise retry
+ commitTable.getClient().completeTransaction(tx1ST);
+
+ TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
+ assertTrue(response.getCommitResponse().getAborted(), "Transaction should abort");
+ assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
+ assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
+ }
+ // ----------------------------------------------------------------------------------------------------------------
+ // The previous 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
+ // (They exercise the communication protocol) TODO Remove???
+ // ----------------------------------------------------------------------------------------------------------------
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private TSOProto.Request createRetryCommitRequest(long ts) {
+ return createCommitRequest(ts, true, testWriteSet);
+ }
+
+ private TSOProto.Request createCommitRequest(long ts, boolean retry, Set<CellId> writeSet) {
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
+ commitBuilder.setStartTimestamp(ts);
+ commitBuilder.setIsRetry(retry);
+ for (CellId cell : writeSet) {
+ commitBuilder.addCellId(cell.getCellId());
+ }
+ return builder.setCommitRequest(commitBuilder.build()).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientResponseHandling.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientResponseHandling.java b/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientResponseHandling.java
new file mode 100644
index 0000000..cf05a9a
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientResponseHandling.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import org.apache.omid.tso.ProgrammableTSOServer;
+import org.apache.omid.tso.ProgrammableTSOServer.AbortResponse;
+import org.apache.omid.tso.ProgrammableTSOServer.CommitResponse;
+import org.apache.omid.tso.ProgrammableTSOServer.TimestampResponse;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.Assert.assertEquals;
+
+public class TestTSOClientResponseHandling {
+
+ private static final int TSO_PORT = 4321;
+ private static final long START_TS = 1L;
+ private static final long COMMIT_TS = 2L;
+
+ private ProgrammableTSOServer tsoServer = new ProgrammableTSOServer(TSO_PORT);
+ // Client under test
+ private TSOClient tsoClient;
+
+ @BeforeClass
+ public void configureAndCreateClient() throws IOException, InterruptedException {
+
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString("localhost:" + TSO_PORT);
+ tsoClient = TSOClient.newInstance(tsoClientConf);
+ }
+
+ @BeforeMethod
+ public void reset() {
+ tsoServer.cleanResponses();
+ }
+
+ @Test
+ public void testTimestampRequestReceivingASuccessfulResponse() throws Exception {
+ // test request timestamp response returns a timestamp
+
+ // Program the TSO to return an ad-hoc Timestamp response
+ tsoServer.queueResponse(new TimestampResponse(START_TS));
+
+ long startTS = tsoClient.getNewStartTimestamp().get();
+ assertEquals(startTS, START_TS);
+ }
+
+ @Test
+ public void testCommitRequestReceivingAnAbortResponse() throws Exception {
+ // test commit request which is aborted on the server side
+ // (e.g. due to conflicts with other transaction) throws an
+ // execution exception with an AbortException as a cause
+
+ // Program the TSO to return an Abort response
+ tsoServer.queueResponse(new AbortResponse(START_TS));
+
+ try {
+ tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
+ } catch (ExecutionException ee) {
+ assertEquals(ee.getCause().getClass(), AbortException.class);
+ }
+ }
+
+ @Test
+ public void testCommitRequestReceivingASuccessfulResponse() throws Exception {
+ // test commit request which is successfully committed on the server
+ // side returns a commit timestamp
+
+ // Program the TSO to return an Commit response (with no required heuristic actions)
+ tsoServer.queueResponse(new CommitResponse(false, START_TS, COMMIT_TS));
+
+ long commitTS = tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
+ assertEquals(commitTS, COMMIT_TS);
+ }
+
+ @Test
+ public void testCommitRequestReceivingAHeuristicResponse() throws Exception {
+ // test commit request which needs heuristic actions from the client
+ // throws an execution exception with a NewTSOException as a cause
+
+ // Program the TSO to return an Commit response requiring heuristic actions
+ tsoServer.queueResponse(new CommitResponse(true, START_TS, COMMIT_TS));
+ try {
+ tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
+ } catch (ExecutionException ee) {
+ assertEquals(ee.getCause().getClass(), NewTSOException.class);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/client/TestUnconnectedTSOClient.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/client/TestUnconnectedTSOClient.java b/tso-server/src/test/java/org/apache/omid/tso/client/TestUnconnectedTSOClient.java
new file mode 100644
index 0000000..883d45f
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/client/TestUnconnectedTSOClient.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import org.apache.omid.tso.util.DummyCellIdImpl;
+import org.apache.statemachine.StateMachine.FsmImpl;
+import org.slf4j.Logger;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static org.slf4j.LoggerFactory.getLogger;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+/**
+ * Test the behavior of requests on a TSOClient component that is not connected to a TSO server.
+ */
+public class TestUnconnectedTSOClient {
+
+ private static final Logger LOG = getLogger(TestUnconnectedTSOClient.class);
+
+ private static final int TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST = 2;
+
+ @Test(timeOut = 30_000) // 30 secs
+ public void testRequestsDoneOnAnUnconnectedTSOClientAlwaysReturn() throws Exception {
+
+ OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
+ tsoClientConf.setConnectionString("localhost:12345");
+ tsoClientConf.setReconnectionDelayInSecs(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST);
+
+ // Component under test
+ TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
+
+ // Internal accessor to fsm
+ FsmImpl fsm = (FsmImpl) tsoClient.fsm;
+
+ assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
+
+ // Test requests to the 3 relevant methods in TSO client
+
+ try {
+ tsoClient.getNewStartTimestamp().get();
+ fail();
+ } catch (ExecutionException e) {
+ LOG.info("Exception expected");
+ assertEquals(e.getCause().getClass(), ConnectionException.class);
+ TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
+ assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
+ }
+
+ try {
+ tsoClient.commit(1, newHashSet(new DummyCellIdImpl(0xdeadbeefL))).get();
+ fail();
+ } catch (ExecutionException e) {
+ LOG.info("Exception expected");
+ assertEquals(e.getCause().getClass(), ConnectionException.class);
+ TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
+ assertEquals(fsm.getState().getClass(), TSOClient.DisconnectedState.class);
+ }
+
+ tsoClient.close().get();
+ LOG.info("No exception expected");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/tso-server/src/test/resources/log4j.properties b/tso-server/src/test/resources/log4j.properties
index 8273243..5f7911e 100644
--- a/tso-server/src/test/resources/log4j.properties
+++ b/tso-server/src/test/resources/log4j.properties
@@ -40,14 +40,13 @@ log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.bookkeeper=FATAL
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.apache.hadoop.ipc=ERROR
-
-log4j.logger.com.yahoo.omid=INFO
-#log4j.logger.com.yahoo.omid.regionserver.TransactionalRegionServer=TRACE
-#log4j.logger.com.yahoo.omid.TestBasicTransaction=TRACE
-#log4j.logger.com.yahoo.omid.client.TSOClient=TRACE
-#log4j.logger.com.yahoo.omid.client.TransactionState=TRACE
-#log4j.logger.com.yahoo.omid.OmidTestBase=TRACE
-#log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=INFO
+log4j.logger.org.apache.omid=INFO
+#log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE
+#log4j.logger.org.apache.omid.TestBasicTransaction=TRACE
+#log4j.logger.org.apache.omid.client.TSOClient=TRACE
+#log4j.logger.org.apache.omid.client.TransactionState=TRACE
+#log4j.logger.org.apache.omid.OmidTestBase=TRACE
+#log4j.logger.org.apache.omid.tso.ThroughputMonitor=INFO
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/resources/test-omid.yml
----------------------------------------------------------------------
diff --git a/tso-server/src/test/resources/test-omid.yml b/tso-server/src/test/resources/test-omid.yml
index 9db7bd6..4729bb1 100644
--- a/tso-server/src/test/resources/test-omid.yml
+++ b/tso-server/src/test/resources/test-omid.yml
@@ -9,13 +9,13 @@ maxBatchSize: 500
batchPersistTimeoutInMs: 100
networkIfaceName: eth1
-commitTableStoreModule: !!com.yahoo.omid.committable.hbase.DefaultHBaseCommitTableStorageModule
+commitTableStoreModule: !!org.apache.omid.committable.hbase.DefaultHBaseCommitTableStorageModule
tableName: "sieve_omid:OMID_TIMESTAMP_F"
-timestampStoreModule: !!com.yahoo.omid.timestamp.storage.DefaultHBaseTimestampStorageModule
+timestampStoreModule: !!org.apache.omid.timestamp.storage.DefaultHBaseTimestampStorageModule
tableName: "sieve_omid:OMID_COMMIT_TABLE_F"
familyName: "MAX_TIMESTAMP_F"
-leaseModule: !!com.yahoo.omid.tso.VoidLeaseManagementModule [ ]
+leaseModule: !!org.apache.omid.tso.VoidLeaseManagementModule [ ]
-metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
\ No newline at end of file
+metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
\ No newline at end of file
[49/52] [abbrv] incubator-omid git commit: [ci skip]prepare for next
development iteration
Posted by ik...@apache.org.
[ci skip]prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/410653e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/410653e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/410653e4
Branch: refs/heads/master
Commit: 410653e47b36508389105f9ef835b0c7920e473a
Parents: 052fb71
Author: Omid CI <om...@yahoo-inc.com>
Authored: Tue Apr 19 21:28:31 2016 +0000
Committer: Omid CI <om...@yahoo-inc.com>
Committed: Tue Apr 19 21:28:31 2016 +0000
----------------------------------------------------------------------
benchmarks/pom.xml | 2 +-
codahale-metrics/pom.xml | 2 +-
commit-table/pom.xml | 2 +-
common/pom.xml | 2 +-
examples/pom.xml | 2 +-
hbase-client/pom.xml | 2 +-
hbase-commit-table/pom.xml | 2 +-
hbase-common/pom.xml | 2 +-
hbase-coprocessor/pom.xml | 2 +-
hbase-shims/hbase-0/pom.xml | 2 +-
hbase-shims/hbase-1/pom.xml | 2 +-
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 2 +-
metrics/pom.xml | 2 +-
pom.xml | 4 ++--
statemachine/pom.xml | 2 +-
timestamp-storage/pom.xml | 2 +-
transaction-client/pom.xml | 2 +-
tso-server/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index 59cebbf..8c9e6b6 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>benchmarks</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index 933749a..1c75987 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>org.apache.omid</groupId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index a0adc3f..39b8d0a 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index e20867e..45bb5d4 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 91dc26f..4486814 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>org.apache.omid</groupId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 74a6c66..71aa718 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index e954ff0..c4e5983 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase-commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 89691d9..659c923 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index c632709..c4564be 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase-coprocessor</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index 950e79f..18b15f2 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase0-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index 12c6e38..2a184e6 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase1-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index a49e02f..36a7f77 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>omid-shims-aggregator</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index cfa693d..3fc2f4c 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>hbase-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index 776925c..496e2e3 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>org.apache.omid</groupId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>metrics</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 10c0062..d74eb0c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
<organization>
<name>Apache Software Foundation</name>
@@ -105,7 +105,7 @@
<connection>scm:git:git://github.com/yahoo/omid.git</connection>
<developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
<url>https://github.com/yahoo/omid</url>
- <tag>omid-0.8.1.37</tag>
+ <tag>master</tag>
</scm>
<distributionManagement>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index 24fc52d..b3201d7 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<name>State Machine</name>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index 066d0e4..1ba1789 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>timestamp-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index fb8bd84..1e0c68c 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>transaction-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/410653e4/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index 20077d8..60532a2 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.37</version>
+ <version>0.8.1.38-SNAPSHOT</version>
</parent>
<artifactId>tso-server</artifactId>
[40/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
deleted file mode 100644
index 53b62eb..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
+++ /dev/null
@@ -1,770 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.base.Predicate;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Multimaps;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.transaction.HBaseTransactionManager.CommitTimestampLocatorImpl;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.KeyValueUtil;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Mutation;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.TimeRange;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
-
-/**
- * Provides transactional methods for accessing and modifying a given snapshot
- * of data identified by an opaque {@link Transaction} object. It mimics the
- * behavior in {@link org.apache.hadoop.hbase.client.HTableInterface}
- */
-public class TTable implements Closeable {
-
- private static Logger LOG = LoggerFactory.getLogger(TTable.class);
-
- private final HTableInterface healerTable;
-
- private HTableInterface table;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Construction
- // ----------------------------------------------------------------------------------------------------------------
-
- public TTable(Configuration conf, byte[] tableName) throws IOException {
- this(new HTable(conf, tableName));
- }
-
- public TTable(String tableName) throws IOException {
- this(HBaseConfiguration.create(), Bytes.toBytes(tableName));
- }
-
- public TTable(Configuration conf, String tableName) throws IOException {
- this(conf, Bytes.toBytes(tableName));
- }
-
- public TTable(HTableInterface hTable) throws IOException {
- table = hTable;
- healerTable = new HTable(table.getConfiguration(), table.getTableName());
- }
-
- public TTable(HTableInterface hTable, HTableInterface healerTable) throws IOException {
- table = hTable;
- this.healerTable = healerTable;
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Closeable implementation
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * Releases any resources held or pending changes in internal buffers.
- *
- * @throws IOException
- * if a remote or network exception occurs.
- */
- @Override
- public void close() throws IOException {
- table.close();
- healerTable.close();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Transactional operations
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * Transactional version of {@link HTableInterface#get(Get get)}
- */
- public Result get(Transaction tx, final Get get) throws IOException {
-
- throwExceptionIfOpSetsTimerange(get);
-
- HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
-
- final long readTimestamp = transaction.getStartTimestamp();
- final Get tsget = new Get(get.getRow()).setFilter(get.getFilter());
- TimeRange timeRange = get.getTimeRange();
- long startTime = timeRange.getMin();
- long endTime = Math.min(timeRange.getMax(), readTimestamp + 1);
- tsget.setTimeRange(startTime, endTime).setMaxVersions(1);
- Map<byte[], NavigableSet<byte[]>> kvs = get.getFamilyMap();
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : kvs.entrySet()) {
- byte[] family = entry.getKey();
- NavigableSet<byte[]> qualifiers = entry.getValue();
- if (qualifiers == null || qualifiers.isEmpty()) {
- tsget.addFamily(family);
- } else {
- for (byte[] qualifier : qualifiers) {
- tsget.addColumn(family, qualifier);
- tsget.addColumn(family, CellUtils.addShadowCellSuffix(qualifier));
- }
- }
- }
- LOG.trace("Initial Get = {}", tsget);
-
- // Return the KVs that belong to the transaction snapshot, ask for more
- // versions if needed
- Result result = table.get(tsget);
- List<Cell> filteredKeyValues = Collections.emptyList();
- if (!result.isEmpty()) {
- filteredKeyValues = filterCellsForSnapshot(result.listCells(), transaction, tsget.getMaxVersions());
- }
-
- return Result.create(filteredKeyValues);
- }
-
- /**
- * Transactional version of {@link HTableInterface#delete(Delete delete)}
- */
- public void delete(Transaction tx, Delete delete) throws IOException {
-
- throwExceptionIfOpSetsTimerange(delete);
-
- HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
-
- final long startTimestamp = transaction.getStartTimestamp();
- boolean issueGet = false;
-
- final Put deleteP = new Put(delete.getRow(), startTimestamp);
- final Get deleteG = new Get(delete.getRow());
- Map<byte[], List<Cell>> fmap = delete.getFamilyCellMap();
- if (fmap.isEmpty()) {
- issueGet = true;
- }
- for (List<Cell> cells : fmap.values()) {
- for (Cell cell : cells) {
- CellUtils.validateCell(cell, startTimestamp);
- switch (KeyValue.Type.codeToType(cell.getTypeByte())) {
- case DeleteColumn:
- deleteP.add(CellUtil.cloneFamily(cell),
- CellUtil.cloneQualifier(cell),
- startTimestamp,
- CellUtils.DELETE_TOMBSTONE);
- transaction.addWriteSetElement(
- new HBaseCellId(table,
- delete.getRow(),
- CellUtil.cloneFamily(cell),
- CellUtil.cloneQualifier(cell),
- cell.getTimestamp()));
- break;
- case DeleteFamily:
- deleteG.addFamily(CellUtil.cloneFamily(cell));
- issueGet = true;
- break;
- case Delete:
- if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP) {
- deleteP.add(CellUtil.cloneFamily(cell),
- CellUtil.cloneQualifier(cell),
- startTimestamp,
- CellUtils.DELETE_TOMBSTONE);
- transaction.addWriteSetElement(
- new HBaseCellId(table,
- delete.getRow(),
- CellUtil.cloneFamily(cell),
- CellUtil.cloneQualifier(cell),
- cell.getTimestamp()));
- break;
- } else {
- throw new UnsupportedOperationException(
- "Cannot delete specific versions on Snapshot Isolation.");
- }
- default:
- break;
- }
- }
- }
- if (issueGet) {
- // It's better to perform a transactional get to avoid deleting more
- // than necessary
- Result result = this.get(transaction, deleteG);
- if (!result.isEmpty()) {
- for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> entryF : result.getMap()
- .entrySet()) {
- byte[] family = entryF.getKey();
- for (Entry<byte[], NavigableMap<Long, byte[]>> entryQ : entryF.getValue().entrySet()) {
- byte[] qualifier = entryQ.getKey();
- deleteP.add(family, qualifier, CellUtils.DELETE_TOMBSTONE);
- transaction.addWriteSetElement(new HBaseCellId(table, delete.getRow(), family, qualifier, transaction.getStartTimestamp()));
- }
- }
- }
- }
-
- if (!deleteP.isEmpty()) {
- table.put(deleteP);
- }
-
- }
-
- /**
- * Transactional version of {@link HTableInterface#put(Put put)}
- */
- public void put(Transaction tx, Put put) throws IOException {
-
- throwExceptionIfOpSetsTimerange(put);
-
- HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
-
- final long startTimestamp = transaction.getStartTimestamp();
- // create put with correct ts
- final Put tsput = new Put(put.getRow(), startTimestamp);
- Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
- for (List<Cell> kvl : kvs.values()) {
- for (Cell c : kvl) {
- CellUtils.validateCell(c, startTimestamp);
- // Reach into keyvalue to update timestamp.
- // It's not nice to reach into keyvalue internals,
- // but we want to avoid having to copy the whole thing
- KeyValue kv = KeyValueUtil.ensureKeyValue(c);
- Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), startTimestamp);
- tsput.add(kv);
-
- transaction.addWriteSetElement(
- new HBaseCellId(table,
- CellUtil.cloneRow(kv),
- CellUtil.cloneFamily(kv),
- CellUtil.cloneQualifier(kv),
- kv.getTimestamp()));
- }
- }
-
- table.put(tsput);
- }
-
- /**
- * Transactional version of {@link HTableInterface#getScanner(Scan scan)}
- */
- public ResultScanner getScanner(Transaction tx, Scan scan) throws IOException {
-
- throwExceptionIfOpSetsTimerange(scan);
-
- HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
-
- Scan tsscan = new Scan(scan);
- tsscan.setMaxVersions(1);
- tsscan.setTimeRange(0, transaction.getStartTimestamp() + 1);
- Map<byte[], NavigableSet<byte[]>> kvs = scan.getFamilyMap();
- for (Map.Entry<byte[], NavigableSet<byte[]>> entry : kvs.entrySet()) {
- byte[] family = entry.getKey();
- NavigableSet<byte[]> qualifiers = entry.getValue();
- if (qualifiers == null) {
- continue;
- }
- for (byte[] qualifier : qualifiers) {
- tsscan.addColumn(family, CellUtils.addShadowCellSuffix(qualifier));
- }
- }
- return new TransactionalClientScanner(transaction, tsscan, 1);
- }
-
- /**
- * Filters the raw results returned from HBase and returns only those
- * belonging to the current snapshot, as defined by the transaction
- * object. If the raw results don't contain enough information for a
- * particular qualifier, it will request more versions from HBase.
- *
- * @param rawCells
- * Raw cells that we are going to filter
- * @param transaction
- * Defines the current snapshot
- * @param versionsToRequest
- * Number of versions requested from hbase
- * @return Filtered KVs belonging to the transaction snapshot
- * @throws IOException
- */
- List<Cell> filterCellsForSnapshot(List<Cell> rawCells, HBaseTransaction transaction,
- int versionsToRequest) throws IOException {
-
- assert (rawCells != null && transaction != null && versionsToRequest >= 1);
-
- List<Cell> keyValuesInSnapshot = new ArrayList<>();
- List<Get> pendingGetsList = new ArrayList<>();
-
- int numberOfVersionsToFetch = versionsToRequest * 2;
- if (numberOfVersionsToFetch < 1) {
- numberOfVersionsToFetch = versionsToRequest;
- }
-
- Map<Long, Long> commitCache = buildCommitCache(rawCells);
-
- for (Collection<Cell> columnCells : groupCellsByColumnFilteringShadowCells(rawCells)) {
- boolean snapshotValueFound = false;
- Cell oldestCell = null;
- for (Cell cell : columnCells) {
- if (isCellInSnapshot(cell, transaction, commitCache)) {
- if (!CellUtil.matchingValue(cell, CellUtils.DELETE_TOMBSTONE)) {
- keyValuesInSnapshot.add(cell);
- }
- snapshotValueFound = true;
- break;
- }
- oldestCell = cell;
- }
- if (!snapshotValueFound) {
- assert (oldestCell != null);
- Get pendingGet = createPendingGet(oldestCell, numberOfVersionsToFetch);
- pendingGetsList.add(pendingGet);
- }
- }
-
- if (!pendingGetsList.isEmpty()) {
- Result[] pendingGetsResults = table.get(pendingGetsList);
- for (Result pendingGetResult : pendingGetsResults) {
- if (!pendingGetResult.isEmpty()) {
- keyValuesInSnapshot.addAll(
- filterCellsForSnapshot(pendingGetResult.listCells(), transaction, numberOfVersionsToFetch));
- }
- }
- }
-
- Collections.sort(keyValuesInSnapshot, KeyValue.COMPARATOR);
-
- assert (keyValuesInSnapshot.size() <= rawCells.size());
- return keyValuesInSnapshot;
- }
-
- private Map<Long, Long> buildCommitCache(List<Cell> rawCells) {
-
- Map<Long, Long> commitCache = new HashMap<>();
-
- for (Cell cell : rawCells) {
- if (CellUtils.isShadowCell(cell)) {
- commitCache.put(cell.getTimestamp(), Bytes.toLong(CellUtil.cloneValue(cell)));
- }
- }
-
- return commitCache;
- }
-
- private boolean isCellInSnapshot(Cell kv, HBaseTransaction transaction, Map<Long, Long> commitCache)
- throws IOException {
-
- long startTimestamp = transaction.getStartTimestamp();
-
- if (kv.getTimestamp() == startTimestamp) {
- return true;
- }
-
- Optional<Long> commitTimestamp =
- tryToLocateCellCommitTimestamp(transaction.getTransactionManager(), transaction.getEpoch(), kv, commitCache);
-
- return commitTimestamp.isPresent() && commitTimestamp.get() < startTimestamp;
- }
-
- private Get createPendingGet(Cell cell, int versionCount) throws IOException {
-
- Get pendingGet = new Get(CellUtil.cloneRow(cell));
- pendingGet.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));
- pendingGet.addColumn(CellUtil.cloneFamily(cell), CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength()));
- pendingGet.setMaxVersions(versionCount);
- pendingGet.setTimeRange(0, cell.getTimestamp());
-
- return pendingGet;
- }
-
- private Optional<Long> tryToLocateCellCommitTimestamp(AbstractTransactionManager transactionManager,
- long epoch,
- Cell cell,
- Map<Long, Long> commitCache)
- throws IOException {
-
- CommitTimestamp tentativeCommitTimestamp =
- transactionManager.locateCellCommitTimestamp(
- cell.getTimestamp(),
- epoch,
- new CommitTimestampLocatorImpl(
- new HBaseCellId(table,
- CellUtil.cloneRow(cell),
- CellUtil.cloneFamily(cell),
- CellUtil.cloneQualifier(cell),
- cell.getTimestamp()),
- commitCache));
-
- // If transaction that added the cell was invalidated
- if (!tentativeCommitTimestamp.isValid()) {
- return Optional.absent();
- }
-
- switch (tentativeCommitTimestamp.getLocation()) {
- case COMMIT_TABLE:
- // If the commit timestamp is found in the persisted commit table,
- // that means the writing process of the shadow cell in the post
- // commit phase of the client probably failed, so we heal the shadow
- // cell with the right commit timestamp for avoiding further reads to
- // hit the storage
- healShadowCell(cell, tentativeCommitTimestamp.getValue());
- case CACHE:
- case SHADOW_CELL:
- return Optional.of(tentativeCommitTimestamp.getValue());
- case NOT_PRESENT:
- return Optional.absent();
- default:
- assert (false);
- return Optional.absent();
- }
- }
-
- void healShadowCell(Cell cell, long commitTimestamp) {
- Put put = new Put(CellUtil.cloneRow(cell));
- byte[] family = CellUtil.cloneFamily(cell);
- byte[] shadowCellQualifier = CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
- cell.getQualifierOffset(),
- cell.getQualifierLength());
- put.add(family, shadowCellQualifier, cell.getTimestamp(), Bytes.toBytes(commitTimestamp));
- try {
- healerTable.put(put);
- } catch (IOException e) {
- LOG.warn("Failed healing shadow cell for kv {}", cell, e);
- }
- }
-
- protected class TransactionalClientScanner implements ResultScanner {
- private HBaseTransaction state;
- private ResultScanner innerScanner;
- private int maxVersions;
-
- TransactionalClientScanner(HBaseTransaction state, Scan scan, int maxVersions)
- throws IOException {
- this.state = state;
- this.innerScanner = table.getScanner(scan);
- this.maxVersions = maxVersions;
- }
-
-
- @Override
- public Result next() throws IOException {
- List<Cell> filteredResult = Collections.emptyList();
- while (filteredResult.isEmpty()) {
- Result result = innerScanner.next();
- if (result == null) {
- return null;
- }
- if (!result.isEmpty()) {
- filteredResult = filterCellsForSnapshot(result.listCells(), state, maxVersions);
- }
- }
- return Result.create(filteredResult);
- }
-
- // In principle no need to override, copied from super.next(int) to make
- // sure it works even if super.next(int)
- // changes its implementation
- @Override
- public Result[] next(int nbRows) throws IOException {
- // Collect values to be returned here
- ArrayList<Result> resultSets = new ArrayList<>(nbRows);
- for (int i = 0; i < nbRows; i++) {
- Result next = next();
- if (next != null) {
- resultSets.add(next);
- } else {
- break;
- }
- }
- return resultSets.toArray(new Result[resultSets.size()]);
- }
-
- @Override
- public void close() {
- innerScanner.close();
- }
-
- @Override
- public Iterator<Result> iterator() {
- return new ResultIterator(this);
- }
-
- // ------------------------------------------------------------------------------------------------------------
- // --------------------------------- Helper class for TransactionalClientScanner ------------------------------
- // ------------------------------------------------------------------------------------------------------------
-
- class ResultIterator implements Iterator<Result> {
-
- TransactionalClientScanner scanner;
- Result currentResult;
-
- ResultIterator(TransactionalClientScanner scanner) {
- try {
- this.scanner = scanner;
- currentResult = scanner.next();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public boolean hasNext() {
- return currentResult != null && !currentResult.isEmpty();
- }
-
- @Override
- public Result next() {
- try {
- Result result = currentResult;
- currentResult = scanner.next();
- return result;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public void remove() {
- throw new RuntimeException("Not implemented");
- }
-
- }
-
- }
-
- /**
- * Delegates to {@link HTable#getTableName()}
- */
- public byte[] getTableName() {
- return table.getTableName();
- }
-
- /**
- * Delegates to {@link HTable#getConfiguration()}
- */
- public Configuration getConfiguration() {
- return table.getConfiguration();
- }
-
- /**
- * Delegates to {@link HTable#getTableDescriptor()}
- */
- public HTableDescriptor getTableDescriptor() throws IOException {
- return table.getTableDescriptor();
- }
-
- /**
- * Transactional version of {@link HTableInterface#exists(Get get)}
- */
- public boolean exists(Transaction transaction, Get get) throws IOException {
- Result result = get(transaction, get);
- return !result.isEmpty();
- }
-
- /* TODO What should we do with this methods???
- * @Override public void batch(Transaction transaction, List<? extends Row>
- * actions, Object[] results) throws IOException, InterruptedException {}
- *
- * @Override public Object[] batch(Transaction transaction, List<? extends
- * Row> actions) throws IOException, InterruptedException {}
- *
- * @Override public <R> void batchCallback(Transaction transaction, List<?
- * extends Row> actions, Object[] results, Callback<R> callback) throws
- * IOException, InterruptedException {}
- *
- * @Override public <R> Object[] batchCallback(List<? extends Row> actions,
- * Callback<R> callback) throws IOException, InterruptedException {}
- */
-
- /**
- * Transactional version of {@link HTableInterface#get(List<Get> gets)}
- */
- public Result[] get(Transaction transaction, List<Get> gets) throws IOException {
- Result[] results = new Result[gets.size()];
- int i = 0;
- for (Get get : gets) {
- results[i++] = get(transaction, get);
- }
- return results;
- }
-
- /**
- * Transactional version of {@link HTableInterface#getScanner(byte[] family)}
- */
- public ResultScanner getScanner(Transaction transaction, byte[] family) throws IOException {
- Scan scan = new Scan();
- scan.addFamily(family);
- return getScanner(transaction, scan);
- }
-
- /**
- * Transactional version of {@link HTableInterface#getScanner(byte[] family, byte[] qualifier)}
- */
- public ResultScanner getScanner(Transaction transaction, byte[] family, byte[] qualifier)
- throws IOException {
- Scan scan = new Scan();
- scan.addColumn(family, qualifier);
- return getScanner(transaction, scan);
- }
-
- /**
- * Transactional version of {@link HTableInterface#put(List<Put> puts)}
- */
- public void put(Transaction transaction, List<Put> puts) throws IOException {
- for (Put put : puts) {
- put(transaction, put);
- }
- }
-
- /**
- * Transactional version of {@link HTableInterface#delete(List<Delete> deletes)}
- */
- public void delete(Transaction transaction, List<Delete> deletes) throws IOException {
- for (Delete delete : deletes) {
- delete(transaction, delete);
- }
- }
-
- /**
- * Provides access to the underliying HTable in order to configure it or to
- * perform unsafe (non-transactional) operations. The latter would break the
- * transactional guarantees of the whole system.
- *
- * @return The underlying HTable object
- */
- public HTableInterface getHTable() {
- return table;
- }
-
- /**
- * Delegates to {@link HTable#setAutoFlush(boolean autoFlush)}
- */
- public void setAutoFlush(boolean autoFlush) {
- table.setAutoFlush(autoFlush, true);
- }
-
- /**
- * Delegates to {@link HTable#isAutoFlush()}
- */
- public boolean isAutoFlush() {
- return table.isAutoFlush();
- }
-
- /**
- * Delegates to {@link HTable.getWriteBufferSize()}
- */
- public long getWriteBufferSize() {
- return table.getWriteBufferSize();
- }
-
- /**
- * Delegates to {@link HTable.setWriteBufferSize()}
- */
- public void setWriteBufferSize(long writeBufferSize) throws IOException {
- table.setWriteBufferSize(writeBufferSize);
- }
-
- /**
- * Delegates to {@link HTable.flushCommits()}
- */
- public void flushCommits() throws IOException {
- table.flushCommits();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private void throwExceptionIfOpSetsTimerange(Get getOperation) {
- TimeRange tr = getOperation.getTimeRange();
- checkTimerangeIsSetToDefaultValuesOrThrowException(tr);
- }
-
- private void throwExceptionIfOpSetsTimerange(Scan scanOperation) {
- TimeRange tr = scanOperation.getTimeRange();
- checkTimerangeIsSetToDefaultValuesOrThrowException(tr);
- }
-
- private void checkTimerangeIsSetToDefaultValuesOrThrowException(TimeRange tr) {
- if (tr.getMin() != 0L || tr.getMax() != Long.MAX_VALUE) {
- throw new IllegalArgumentException(
- "Timestamp/timerange not allowed in transactional user operations");
- }
- }
-
- private void throwExceptionIfOpSetsTimerange(Mutation userOperation) {
- if (userOperation.getTimeStamp() != HConstants.LATEST_TIMESTAMP) {
- throw new IllegalArgumentException(
- "Timestamp not allowed in transactional user operations");
- }
- }
-
- private HBaseTransaction enforceHBaseTransactionAsParam(Transaction tx) {
- if (tx instanceof HBaseTransaction) {
- return (HBaseTransaction) tx;
- } else {
- throw new IllegalArgumentException(
- String.format("The transaction object passed %s is not an instance of HBaseTransaction",
- tx.getClass().getName()));
- }
- }
-
- static ImmutableList<Collection<Cell>> groupCellsByColumnFilteringShadowCells(List<Cell> rawCells) {
-
- Predicate<Cell> shadowCellFilter = new Predicate<Cell>() {
-
- @Override
- public boolean apply(Cell cell) {
- return !CellUtils.isShadowCell(cell);
- }
-
- };
-
- Function<Cell, ColumnWrapper> cellToColumnWrapper = new Function<Cell, ColumnWrapper>() {
-
- @Override
- public ColumnWrapper apply(Cell cell) {
- return new ColumnWrapper(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));
- }
-
- };
-
- return Multimaps.index(Iterables.filter(rawCells, shadowCellFilter), cellToColumnWrapper)
- .asMap().values()
- .asList();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
deleted file mode 100644
index 77548a5..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.yahoo.omid.transaction.CellUtils.CellGetter;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-
-import java.io.IOException;
-
-public class TTableCellGetterAdapter implements CellGetter {
-
- private final TTable txTable;
-
- public TTableCellGetterAdapter(TTable txTable) {
- this.txTable = txTable;
- }
-
- public Result get(Get get) throws IOException {
- return txTable.getHTable().get(get);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/ColumnWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/ColumnWrapper.java b/hbase-client/src/main/java/org/apache/omid/transaction/ColumnWrapper.java
new file mode 100644
index 0000000..012d52e
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/ColumnWrapper.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import java.util.Arrays;
+
+public class ColumnWrapper {
+ private byte[] family;
+ private byte[] qualifier;
+
+ public ColumnWrapper(byte[] family, byte[] qualifier) {
+ this.family = family;
+ this.qualifier = qualifier;
+ }
+
+ public byte[] getFamily() {
+ return family;
+ }
+
+ public byte[] getQualifier() {
+ return qualifier;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(family);
+ result = prime * result + Arrays.hashCode(qualifier);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ ColumnWrapper other = (ColumnWrapper) obj;
+ if (!Arrays.equals(family, other.family))
+ return false;
+ if (!Arrays.equals(qualifier, other.qualifier))
+ return false;
+ return true;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseAsyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseAsyncPostCommitter.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseAsyncPostCommitter.java
new file mode 100644
index 0000000..1037165
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseAsyncPostCommitter.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import org.apache.omid.tso.client.CellId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.Callable;
+
+public class HBaseAsyncPostCommitter implements PostCommitActions {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseAsyncPostCommitter.class);
+
+ private PostCommitActions syncPostCommitter;
+
+ private ListeningExecutorService postCommitExecutor;
+
+ public HBaseAsyncPostCommitter(PostCommitActions postCommitter, ListeningExecutorService postCommitExecutor) {
+ this.syncPostCommitter = postCommitter;
+ this.postCommitExecutor = postCommitExecutor;
+ }
+
+ @Override
+ public ListenableFuture<Void> updateShadowCells(final AbstractTransaction<? extends CellId> transaction) {
+
+ return postCommitExecutor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ syncPostCommitter.updateShadowCells(transaction);
+ return null;
+ }
+
+ });
+
+ }
+
+ @Override
+ public ListenableFuture<Void> removeCommitTableEntry(final AbstractTransaction<? extends CellId> transaction) {
+
+ return postCommitExecutor.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ syncPostCommitter.removeCommitTableEntry(transaction);
+ return null;
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseCellId.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseCellId.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseCellId.java
new file mode 100644
index 0000000..1f4add1
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseCellId.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.hash.Hashing;
+import org.apache.omid.tso.client.CellId;
+import org.apache.hadoop.hbase.client.HTableInterface;
+
+import static com.google.common.base.Charsets.UTF_8;
+
+public class HBaseCellId implements CellId {
+
+ private final HTableInterface table;
+ private final byte[] row;
+ private final byte[] family;
+ private final byte[] qualifier;
+ private long timestamp;
+
+ public HBaseCellId(HTableInterface table, byte[] row, byte[] family, byte[] qualifier, long timestamp) {
+ this.timestamp = timestamp;
+ this.table = table;
+ this.row = row;
+ this.family = family;
+ this.qualifier = qualifier;
+ }
+
+ public HTableInterface getTable() {
+ return table;
+ }
+
+ public byte[] getRow() {
+ return row;
+ }
+
+ public byte[] getFamily() {
+ return family;
+ }
+
+ public byte[] getQualifier() {
+ return qualifier;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public String toString() {
+ return new String(table.getTableName(), UTF_8)
+ + ":" + new String(row, UTF_8)
+ + ":" + new String(family, UTF_8)
+ + ":" + new String(qualifier, UTF_8)
+ + ":" + timestamp;
+ }
+
+ @Override
+ public long getCellId() {
+ return Hashing.murmur3_128().newHasher()
+ .putBytes(table.getTableName())
+ .putBytes(row)
+ .putBytes(family)
+ .putBytes(qualifier)
+ .hash().asLong();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseOmidClientConfiguration.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseOmidClientConfiguration.java
new file mode 100644
index 0000000..822f8f5
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseOmidClientConfiguration.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import org.apache.omid.YAMLUtils;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+import org.apache.omid.tso.client.OmidClientConfiguration.PostCommitMode;
+import org.apache.omid.tso.client.OmidClientConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+/**
+ * Configuration for HBase's Omid client side
+ */
+public class HBaseOmidClientConfiguration extends SecureHBaseConfig {
+
+ private static final String DEFAULT_CONFIG_FILE_NAME = "default-hbase-omid-client-config.yml";
+ private static final String CONFIG_FILE_NAME = "hbase-omid-client-config.yml";
+ private Configuration hbaseConfiguration = HBaseConfiguration.create();
+ private String commitTableName;
+ @Inject
+ private OmidClientConfiguration omidClientConfiguration;
+ private MetricsRegistry metrics;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Instantiation
+ // ----------------------------------------------------------------------------------------------------------------
+ public HBaseOmidClientConfiguration() {
+ this(CONFIG_FILE_NAME);
+ }
+
+ @VisibleForTesting
+ HBaseOmidClientConfiguration(String configFileName) {
+ new YAMLUtils().loadSettings(configFileName, DEFAULT_CONFIG_FILE_NAME, this);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters and setters for config params
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public Configuration getHBaseConfiguration() {
+ return hbaseConfiguration;
+ }
+
+ public void setHBaseConfiguration(Configuration hbaseConfiguration) {
+ this.hbaseConfiguration = hbaseConfiguration;
+ }
+
+ public PostCommitMode getPostCommitMode() {
+ return omidClientConfiguration.getPostCommitMode();
+ }
+
+ public void setPostCommitMode(PostCommitMode postCommitMode) {
+ omidClientConfiguration.setPostCommitMode(postCommitMode);
+ }
+
+ public String getCommitTableName() {
+ return commitTableName;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.hbase.commitTableName")
+ public void setCommitTableName(String commitTableName) {
+ this.commitTableName = commitTableName;
+ }
+
+ public OmidClientConfiguration getOmidClientConfiguration() {
+ return omidClientConfiguration;
+ }
+
+ public void setOmidClientConfiguration(OmidClientConfiguration omidClientConfiguration) {
+ this.omidClientConfiguration = omidClientConfiguration;
+ }
+
+ public MetricsRegistry getMetrics() {
+ return metrics;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.hbase.metrics")
+ public void setMetrics(MetricsRegistry metrics) {
+ this.metrics = metrics;
+ }
+
+ // Delegation to make end-user life better
+
+ public OmidClientConfiguration.ConnType getConnectionType() {
+ return omidClientConfiguration.getConnectionType();
+ }
+
+ public void setReconnectionDelayInSecs(int reconnectionDelayInSecs) {
+ omidClientConfiguration.setReconnectionDelayInSecs(reconnectionDelayInSecs);
+ }
+
+ public void setExecutorThreads(int executorThreads) {
+ omidClientConfiguration.setExecutorThreads(executorThreads);
+ }
+
+ public int getRequestTimeoutInMs() {
+ return omidClientConfiguration.getRequestTimeoutInMs();
+ }
+
+ public void setConnectionString(String connectionString) {
+ omidClientConfiguration.setConnectionString(connectionString);
+ }
+
+ public void setRequestTimeoutInMs(int requestTimeoutInMs) {
+ omidClientConfiguration.setRequestTimeoutInMs(requestTimeoutInMs);
+ }
+
+ public void setZkConnectionTimeoutInSecs(int zkConnectionTimeoutInSecs) {
+ omidClientConfiguration.setZkConnectionTimeoutInSecs(zkConnectionTimeoutInSecs);
+ }
+
+ public void setConnectionType(OmidClientConfiguration.ConnType connectionType) {
+ omidClientConfiguration.setConnectionType(connectionType);
+ }
+
+ public void setRequestMaxRetries(int requestMaxRetries) {
+ omidClientConfiguration.setRequestMaxRetries(requestMaxRetries);
+ }
+
+ public int getZkConnectionTimeoutInSecs() {
+ return omidClientConfiguration.getZkConnectionTimeoutInSecs();
+ }
+
+ public void setRetryDelayInMs(int retryDelayInMs) {
+ omidClientConfiguration.setRetryDelayInMs(retryDelayInMs);
+ }
+
+ public int getExecutorThreads() {
+ return omidClientConfiguration.getExecutorThreads();
+ }
+
+ public int getRetryDelayInMs() {
+ return omidClientConfiguration.getRetryDelayInMs();
+ }
+
+ public String getConnectionString() {
+ return omidClientConfiguration.getConnectionString();
+ }
+
+ public int getRequestMaxRetries() {
+ return omidClientConfiguration.getRequestMaxRetries();
+ }
+
+ public int getReconnectionDelayInSecs() {
+ return omidClientConfiguration.getReconnectionDelayInSecs();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java
new file mode 100644
index 0000000..ba657fd
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.Timer;
+import org.apache.omid.tso.client.CellId;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.omid.metrics.MetricsUtils.name;
+
+public class HBaseSyncPostCommitter implements PostCommitActions {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseSyncPostCommitter.class);
+
+ private final MetricsRegistry metrics;
+ private final CommitTable.Client commitTableClient;
+
+ private final Timer commitTableUpdateTimer;
+ private final Timer shadowCellsUpdateTimer;
+
+ public HBaseSyncPostCommitter(MetricsRegistry metrics, CommitTable.Client commitTableClient) {
+ this.metrics = metrics;
+ this.commitTableClient = commitTableClient;
+
+ this.commitTableUpdateTimer = metrics.timer(name("omid", "tm", "hbase", "commitTableUpdate", "latency"));
+ this.shadowCellsUpdateTimer = metrics.timer(name("omid", "tm", "hbase", "shadowCellsUpdate", "latency"));
+ }
+
+ @Override
+ public ListenableFuture<Void> updateShadowCells(AbstractTransaction<? extends CellId> transaction) {
+
+ SettableFuture<Void> updateSCFuture = SettableFuture.create();
+
+ HBaseTransaction tx = HBaseTransactionManager.enforceHBaseTransactionAsParam(transaction);
+
+ shadowCellsUpdateTimer.start();
+ try {
+
+ // Add shadow cells
+ for (HBaseCellId cell : tx.getWriteSet()) {
+ Put put = new Put(cell.getRow());
+ put.add(cell.getFamily(),
+ CellUtils.addShadowCellSuffix(cell.getQualifier(), 0, cell.getQualifier().length),
+ tx.getStartTimestamp(),
+ Bytes.toBytes(tx.getCommitTimestamp()));
+ try {
+ cell.getTable().put(put);
+ } catch (IOException e) {
+ LOG.warn("{}: Error inserting shadow cell {}", tx, cell, e);
+ updateSCFuture.setException(
+ new TransactionManagerException(tx + ": Error inserting shadow cell " + cell, e));
+ }
+ }
+
+ // Flush affected tables before returning to avoid loss of shadow cells updates when autoflush is disabled
+ try {
+ tx.flushTables();
+ updateSCFuture.set(null);
+ } catch (IOException e) {
+ LOG.warn("{}: Error while flushing writes", tx, e);
+ updateSCFuture.setException(new TransactionManagerException(tx + ": Error while flushing writes", e));
+ }
+
+ } finally {
+ shadowCellsUpdateTimer.stop();
+ }
+
+ return updateSCFuture;
+
+ }
+
+ @Override
+ public ListenableFuture<Void> removeCommitTableEntry(AbstractTransaction<? extends CellId> transaction) {
+
+ SettableFuture<Void> updateSCFuture = SettableFuture.create();
+
+ HBaseTransaction tx = HBaseTransactionManager.enforceHBaseTransactionAsParam(transaction);
+
+ commitTableUpdateTimer.start();
+
+ try {
+ commitTableClient.completeTransaction(tx.getStartTimestamp()).get();
+ updateSCFuture.set(null);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.warn("{}: interrupted during commit table entry delete", tx, e);
+ updateSCFuture.setException(
+ new TransactionManagerException(tx + ": interrupted during commit table entry delete"));
+ } catch (ExecutionException e) {
+ LOG.warn("{}: can't remove commit table entry", tx, e);
+ updateSCFuture.setException(new TransactionManagerException(tx + ": can't remove commit table entry"));
+ } finally {
+ commitTableUpdateTimer.stop();
+ }
+
+ return updateSCFuture;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java
new file mode 100644
index 0000000..6e00de0
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+public class HBaseTransaction extends AbstractTransaction<HBaseCellId> {
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseTransaction.class);
+
+ HBaseTransaction(long transactionId, long epoch, Set<HBaseCellId> writeSet, AbstractTransactionManager tm) {
+ super(transactionId, epoch, writeSet, tm);
+ }
+
+ @Override
+ public void cleanup() {
+ Set<HBaseCellId> writeSet = getWriteSet();
+ for (final HBaseCellId cell : writeSet) {
+ Delete delete = new Delete(cell.getRow());
+ delete.deleteColumn(cell.getFamily(), cell.getQualifier(), getStartTimestamp());
+ try {
+ cell.getTable().delete(delete);
+ } catch (IOException e) {
+ LOG.warn("Failed cleanup cell {} for Tx {}. This issue has been ignored", new Object[]{cell, getTransactionId(), e});
+ }
+ }
+ try {
+ flushTables();
+ } catch (IOException e) {
+ LOG.warn("Failed flushing tables for Tx {}", getTransactionId(), e);
+ }
+ }
+
+ /**
+ * Flushes pending operations for tables touched by transaction
+ */
+ public void flushTables() throws IOException {
+
+ for (HTableInterface writtenTable : getWrittenTables()) {
+ writtenTable.flushCommits();
+ }
+
+ }
+
+ // ****************************************************************************************************************
+ // Helper methods
+ // ****************************************************************************************************************
+
+ private Set<HTableInterface> getWrittenTables() {
+ HashSet<HBaseCellId> writeSet = (HashSet<HBaseCellId>) getWriteSet();
+ Set<HTableInterface> tables = new HashSet<HTableInterface>();
+ for (HBaseCellId cell : writeSet) {
+ tables.add(cell.getTable());
+ }
+ return tables;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionClient.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionClient.java
new file mode 100644
index 0000000..aaf236b
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionClient.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+public interface HBaseTransactionClient {
+ boolean isCommitted(HBaseCellId hBaseCellId) throws TransactionException;
+
+ long getLowWatermark() throws TransactionException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionManager.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionManager.java
new file mode 100644
index 0000000..fd45dc2
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransactionManager.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.committable.hbase.HBaseCommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.tools.hbase.HBaseLogin;
+import org.apache.omid.tso.client.CellId;
+import org.apache.omid.tso.client.TSOClient;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+
+public class HBaseTransactionManager extends AbstractTransactionManager implements HBaseTransactionClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseTransactionManager.class);
+
+ private static class HBaseTransactionFactory implements TransactionFactory<HBaseCellId> {
+
+ @Override
+ public HBaseTransaction createTransaction(long transactionId, long epoch, AbstractTransactionManager tm) {
+
+ return new HBaseTransaction(transactionId, epoch, new HashSet<HBaseCellId>(), tm);
+
+ }
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Construction
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public static TransactionManager newInstance() throws IOException, InterruptedException {
+ return newInstance(new HBaseOmidClientConfiguration());
+ }
+
+ public static TransactionManager newInstance(HBaseOmidClientConfiguration configuration)
+ throws IOException, InterruptedException {
+ //Logging in to Secure HBase if required
+ HBaseLogin.loginIfNeeded(configuration);
+ return builder(configuration).build();
+ }
+
+ @VisibleForTesting
+ static class Builder {
+
+ // Required parameters
+ private final HBaseOmidClientConfiguration hbaseOmidClientConf;
+
+ // Optional parameters - initialized to default values
+ private Optional<TSOClient> tsoClient = Optional.absent();
+ private Optional<CommitTable.Client> commitTableClient = Optional.absent();
+ private Optional<PostCommitActions> postCommitter = Optional.absent();
+
+ private Builder(HBaseOmidClientConfiguration hbaseOmidClientConf) {
+ this.hbaseOmidClientConf = hbaseOmidClientConf;
+ }
+
+ Builder tsoClient(TSOClient tsoClient) {
+ this.tsoClient = Optional.of(tsoClient);
+ return this;
+ }
+
+ Builder commitTableClient(CommitTable.Client client) {
+ this.commitTableClient = Optional.of(client);
+ return this;
+ }
+
+ Builder postCommitter(PostCommitActions postCommitter) {
+ this.postCommitter = Optional.of(postCommitter);
+ return this;
+ }
+
+ HBaseTransactionManager build() throws IOException, InterruptedException {
+
+ CommitTable.Client commitTableClient = this.commitTableClient.or(buildCommitTableClient()).get();
+ PostCommitActions postCommitter = this.postCommitter.or(buildPostCommitter(commitTableClient)).get();
+ TSOClient tsoClient = this.tsoClient.or(buildTSOClient()).get();
+
+ return new HBaseTransactionManager(hbaseOmidClientConf,
+ postCommitter,
+ tsoClient,
+ commitTableClient,
+ new HBaseTransactionFactory());
+ }
+
+ private Optional<TSOClient> buildTSOClient() throws IOException, InterruptedException {
+ return Optional.of(TSOClient.newInstance(hbaseOmidClientConf.getOmidClientConfiguration()));
+ }
+
+
+ private Optional<CommitTable.Client> buildCommitTableClient() throws IOException {
+ HBaseCommitTableConfig commitTableConf = new HBaseCommitTableConfig();
+ commitTableConf.setTableName(hbaseOmidClientConf.getCommitTableName());
+ CommitTable commitTable = new HBaseCommitTable(hbaseOmidClientConf.getHBaseConfiguration(), commitTableConf);
+ return Optional.of(commitTable.getClient());
+ }
+
+ private Optional<PostCommitActions> buildPostCommitter(CommitTable.Client commitTableClient ) {
+
+ PostCommitActions postCommitter;
+ PostCommitActions syncPostCommitter = new HBaseSyncPostCommitter(hbaseOmidClientConf.getMetrics(),
+ commitTableClient);
+ switch(hbaseOmidClientConf.getPostCommitMode()) {
+ case ASYNC:
+ ListeningExecutorService postCommitExecutor =
+ MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
+ postCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
+ break;
+ case SYNC:
+ default:
+ postCommitter = syncPostCommitter;
+ break;
+ }
+
+ return Optional.of(postCommitter);
+ }
+
+ }
+
+ @VisibleForTesting
+ static Builder builder(HBaseOmidClientConfiguration hbaseOmidClientConf) {
+ return new Builder(hbaseOmidClientConf);
+ }
+
+ private HBaseTransactionManager(HBaseOmidClientConfiguration hBaseOmidClientConfiguration,
+ PostCommitActions postCommitter,
+ TSOClient tsoClient,
+ CommitTable.Client commitTableClient,
+ HBaseTransactionFactory hBaseTransactionFactory) {
+
+ super(hBaseOmidClientConfiguration.getMetrics(),
+ postCommitter,
+ tsoClient,
+ commitTableClient,
+ hBaseTransactionFactory);
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // AbstractTransactionManager overwritten methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public void preCommit(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {
+ try {
+ // Flush all pending writes
+ HBaseTransaction hBaseTx = enforceHBaseTransactionAsParam(transaction);
+ hBaseTx.flushTables();
+ } catch (IOException e) {
+ throw new TransactionManagerException("Exception while flushing writes", e);
+ }
+ }
+
+ @Override
+ public void preRollback(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {
+ try {
+ // Flush all pending writes
+ HBaseTransaction hBaseTx = enforceHBaseTransactionAsParam(transaction);
+ hBaseTx.flushTables();
+ } catch (IOException e) {
+ throw new TransactionManagerException("Exception while flushing writes", e);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // HBaseTransactionClient method implementations
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public boolean isCommitted(HBaseCellId hBaseCellId) throws TransactionException {
+ try {
+ CommitTimestamp tentativeCommitTimestamp =
+ locateCellCommitTimestamp(hBaseCellId.getTimestamp(), tsoClient.getEpoch(),
+ new CommitTimestampLocatorImpl(hBaseCellId, Maps.<Long, Long>newHashMap()));
+
+ // If transaction that added the cell was invalidated
+ if (!tentativeCommitTimestamp.isValid()) {
+ return false;
+ }
+
+ switch (tentativeCommitTimestamp.getLocation()) {
+ case COMMIT_TABLE:
+ case SHADOW_CELL:
+ return true;
+ case NOT_PRESENT:
+ return false;
+ case CACHE: // cache was empty
+ default:
+ return false;
+ }
+ } catch (IOException e) {
+ throw new TransactionException("Failure while checking if a transaction was committed", e);
+ }
+ }
+
+ @Override
+ public long getLowWatermark() throws TransactionException {
+ try {
+ return commitTableClient.readLowWatermark().get();
+ } catch (ExecutionException ee) {
+ throw new TransactionException("Error reading low watermark", ee.getCause());
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new TransactionException("Interrupted reading low watermark", ie);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ static HBaseTransaction enforceHBaseTransactionAsParam(AbstractTransaction<? extends CellId> tx) {
+
+ if (tx instanceof HBaseTransaction) {
+ return (HBaseTransaction) tx;
+ } else {
+ throw new IllegalArgumentException(
+ "The transaction object passed is not an instance of HBaseTransaction");
+ }
+
+ }
+
+ static class CommitTimestampLocatorImpl implements CommitTimestampLocator {
+
+ private HBaseCellId hBaseCellId;
+ private final Map<Long, Long> commitCache;
+
+ CommitTimestampLocatorImpl(HBaseCellId hBaseCellId, Map<Long, Long> commitCache) {
+ this.hBaseCellId = hBaseCellId;
+ this.commitCache = commitCache;
+ }
+
+ @Override
+ public Optional<Long> readCommitTimestampFromCache(long startTimestamp) {
+ if (commitCache.containsKey(startTimestamp)) {
+ return Optional.of(commitCache.get(startTimestamp));
+ }
+ return Optional.absent();
+ }
+
+ @Override
+ public Optional<Long> readCommitTimestampFromShadowCell(long startTimestamp) throws IOException {
+
+ Get get = new Get(hBaseCellId.getRow());
+ byte[] family = hBaseCellId.getFamily();
+ byte[] shadowCellQualifier = CellUtils.addShadowCellSuffix(hBaseCellId.getQualifier());
+ get.addColumn(family, shadowCellQualifier);
+ get.setMaxVersions(1);
+ get.setTimeStamp(startTimestamp);
+ Result result = hBaseCellId.getTable().get(get);
+ if (result.containsColumn(family, shadowCellQualifier)) {
+ return Optional.of(Bytes.toLong(result.getValue(family, shadowCellQualifier)));
+ }
+ return Optional.absent();
+ }
+
+ }
+
+}
[25/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java
deleted file mode 100644
index d50aac3..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-public interface CellId {
-
- public long getCellId();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java
deleted file mode 100644
index 5c9cd23..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-/**
- * Thrown when an error is produced when performing the actions required
- * to close the communication with the TSO server
- */
-public class ClosingException extends Exception {
-
- private static final long serialVersionUID = -5681694952053689884L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java
deleted file mode 100644
index e7ba0a0..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import java.io.IOException;
-
-/**
- * Thrown when there are problems with the comm channel with the TSO server
- * (e.g. when it is closed or disconnected)
- */
-public class ConnectionException extends IOException {
-
- private static final long serialVersionUID = -8489539195700267443L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java
deleted file mode 100644
index 8a7089c..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.util.concurrent.ListenableFuture;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-public class ForwardingTSOFuture<T> implements TSOFuture<T> {
- private final ListenableFuture<T> future;
-
- public ForwardingTSOFuture(ListenableFuture<T> future) {
- this.future = future;
- }
-
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- return future.cancel(mayInterruptIfRunning);
- }
-
- @Override
- public boolean isCancelled() {
- return future.isCancelled();
- }
-
- @Override
- public boolean isDone() {
- return future.isDone();
- }
-
- @Override
- public T get() throws InterruptedException, ExecutionException {
- return future.get();
- }
-
- @Override
- public T get(long timeout, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- return future.get(timeout, unit);
- }
-
- @Override
- public void addListener(Runnable listener, Executor executor) {
- future.addListener(listener, executor);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java
deleted file mode 100644
index 6d2764a..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-/**
- * Thrown when some incompatibilities between the TSO client & server are
- * found
- */
-public class HandshakeFailedException extends Exception {
-
- private static final long serialVersionUID = 8545505066920548834L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java
deleted file mode 100644
index 5b67441..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable;
-
-import java.io.IOException;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class MockTSOClient implements TSOProtocol {
- private final AtomicLong timestampGenerator = new AtomicLong();
- private final int CONFLICT_MAP_SIZE = 1 * 1000 * 1000;
- private final long[] conflictMap = new long[CONFLICT_MAP_SIZE];
- private final AtomicLong lwm = new AtomicLong();
-
- private final CommitTable.Writer commitTable;
-
- public MockTSOClient(CommitTable.Writer commitTable) {
- this.commitTable = commitTable;
- }
-
- @Override
- public TSOFuture<Long> getNewStartTimestamp() {
- synchronized (conflictMap) {
- SettableFuture<Long> f = SettableFuture.<Long>create();
- f.set(timestampGenerator.incrementAndGet());
- return new ForwardingTSOFuture<Long>(f);
- }
- }
-
- @Override
- public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
- synchronized (conflictMap) {
- SettableFuture<Long> f = SettableFuture.<Long>create();
- if (transactionId < lwm.get()) {
- f.setException(new AbortException());
- return new ForwardingTSOFuture<Long>(f);
- }
-
- boolean canCommit = true;
- for (CellId c : cells) {
- int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
- if (conflictMap[index] >= transactionId) {
- canCommit = false;
- break;
- }
- }
-
- if (canCommit) {
- long commitTimestamp = timestampGenerator.incrementAndGet();
- for (CellId c : cells) {
- int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
- long oldVal = conflictMap[index];
- conflictMap[index] = commitTimestamp;
- long curLwm = lwm.get();
- while (oldVal > curLwm) {
- if (lwm.compareAndSet(curLwm, oldVal)) {
- break;
- }
- curLwm = lwm.get();
- }
- }
-
- f.set(commitTimestamp);
- try {
- commitTable.addCommittedTransaction(transactionId, commitTimestamp);
- commitTable.updateLowWatermark(lwm.get());
- commitTable.flush();
- } catch (IOException ioe) {
- f.setException(ioe);
- }
- } else {
- f.setException(new AbortException());
- }
- return new ForwardingTSOFuture<Long>(f);
- }
- }
-
- @Override
- public TSOFuture<Void> close() {
- SettableFuture<Void> f = SettableFuture.<Void>create();
- f.set(null);
- return new ForwardingTSOFuture<Void>(f);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java
deleted file mode 100644
index dac86e3..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-/**
- * Thrown when a new TSO has been detected
- */
-public class NewTSOException extends Exception {
-
- private static final long serialVersionUID = -3250655858200759321L;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java
deleted file mode 100644
index ad37889..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
-import com.yahoo.omid.YAMLUtils;
-
-/**
- * Configuration for Omid client side
- */
-public class OmidClientConfiguration {
-
- private static final String DEFAULT_CONFIG_FILE_NAME = "omid-client-config.yml";
-
- public enum ConnType {DIRECT, HA}
-
- public enum PostCommitMode {SYNC, ASYNC}
-
- // Basic connection related params
-
- private ConnType connectionType = ConnType.DIRECT;
- private String connectionString;
- private String zkCurrentTsoPath;
- private String zkNamespace;
- private int zkConnectionTimeoutInSecs;
-
- // Communication protocol related params
-
- private int requestMaxRetries;
- private int requestTimeoutInMs;
- private int reconnectionDelayInSecs;
- private int retryDelayInMs;
- private int executorThreads;
-
- // Transaction Manager related params
-
- private PostCommitMode postCommitMode = PostCommitMode.SYNC;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Instantiation
- // ----------------------------------------------------------------------------------------------------------------
-
- public OmidClientConfiguration() {
- new YAMLUtils().loadSettings(DEFAULT_CONFIG_FILE_NAME, this);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters and setters for config params
- // ----------------------------------------------------------------------------------------------------------------
-
- public ConnType getConnectionType() {
- return connectionType;
- }
-
- @Inject(optional = true)
- @Named("omid.client.connectionType")
- public void setConnectionType(ConnType connectionType) {
- this.connectionType = connectionType;
- }
-
- public String getConnectionString() {
- return connectionString;
- }
-
- @Inject(optional = true)
- @Named("omid.client.connectionString")
- public void setConnectionString(String connectionString) {
- this.connectionString = connectionString;
- }
-
- public int getZkConnectionTimeoutInSecs() {
- return zkConnectionTimeoutInSecs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.zkConnectionTimeoutInSecs")
- public void setZkConnectionTimeoutInSecs(int zkConnectionTimeoutInSecs) {
- this.zkConnectionTimeoutInSecs = zkConnectionTimeoutInSecs;
- }
-
- public int getRequestMaxRetries() {
- return requestMaxRetries;
- }
-
- @Inject(optional = true)
- @Named("omid.client.requestMaxRetries")
- public void setRequestMaxRetries(int requestMaxRetries) {
- this.requestMaxRetries = requestMaxRetries;
- }
-
- public int getRequestTimeoutInMs() {
- return requestTimeoutInMs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.requestTimeoutInMs")
- public void setRequestTimeoutInMs(int requestTimeoutInMs) {
- this.requestTimeoutInMs = requestTimeoutInMs;
- }
-
- public int getReconnectionDelayInSecs() {
- return reconnectionDelayInSecs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.reconnectionDelayInSecs")
- public void setReconnectionDelayInSecs(int reconnectionDelayInSecs) {
- this.reconnectionDelayInSecs = reconnectionDelayInSecs;
- }
-
- public int getRetryDelayInMs() {
- return retryDelayInMs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.retryDelayInMs")
- public void setRetryDelayInMs(int retryDelayInMs) {
- this.retryDelayInMs = retryDelayInMs;
- }
-
- public int getExecutorThreads() {
- return executorThreads;
- }
-
- @Inject(optional = true)
- @Named("omid.client.executorThreads")
- public void setExecutorThreads(int executorThreads) {
- this.executorThreads = executorThreads;
- }
-
- public String getZkCurrentTsoPath() {
- return zkCurrentTsoPath;
- }
-
- @Inject(optional = true)
- @Named("omid.ha.zkCurrentTsoPath")
- public void setZkCurrentTsoPath(String zkCurrentTsoPath) {
- this.zkCurrentTsoPath = zkCurrentTsoPath;
- }
-
- public String getZkNamespace() {
- return zkNamespace;
- }
-
- @Inject(optional = true)
- @Named("omid.ha.zkNamespace")
- public void setZkNamespace(String zkNamespace) {
- this.zkNamespace = zkNamespace;
- }
-
- public PostCommitMode getPostCommitMode() {
- return postCommitMode;
- }
-
- @Inject(optional = true)
- @Named("omid.tm.postCommitMode")
- public void setPostCommitMode(PostCommitMode postCommitMode) {
- this.postCommitMode = postCommitMode;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java
deleted file mode 100644
index 4e0c9f7..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-/**
- * Thrown when the requests from TSO client to the TSO server have reached
- * a number of retries
- */
-public class ServiceUnavailableException extends Exception {
-
- private static final long serialVersionUID = -1551974284011474385L;
-
- public ServiceUnavailableException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java
deleted file mode 100644
index dc2307c..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java
+++ /dev/null
@@ -1,933 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.google.common.base.Charsets;
-import com.google.common.net.HostAndPort;
-import com.google.common.util.concurrent.AbstractFuture;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.zk.ZKUtils;
-import com.yahoo.statemachine.StateMachine;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.cache.ChildData;
-import org.apache.curator.framework.recipes.cache.NodeCache;
-import org.apache.curator.framework.recipes.cache.NodeCacheListener;
-import org.jboss.netty.bootstrap.ClientBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
-import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
-import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
-import org.jboss.netty.util.HashedWheelTimer;
-import org.jboss.netty.util.Timeout;
-import org.jboss.netty.util.TimerTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayDeque;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Describes the abstract methods to communicate to the TSO server
- */
-public class TSOClient implements TSOProtocol, NodeCacheListener {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOClient.class);
-
- // Basic configuration constants & defaults TODO: Move DEFAULT_ZK_CLUSTER to a conf class???
- public static final String DEFAULT_ZK_CLUSTER = "localhost:2181";
-
- private static final long DEFAULT_EPOCH = -1L;
- private volatile long epoch = DEFAULT_EPOCH;
-
- // Attributes
- private CuratorFramework zkClient;
- private NodeCache currentTSOZNode;
-
- private ChannelFactory factory;
- private ClientBootstrap bootstrap;
- private Channel currentChannel;
- private final ScheduledExecutorService fsmExecutor;
- StateMachine.Fsm fsm;
-
- private final int requestTimeoutInMs;
- private final int requestMaxRetries;
- private final int tsoReconnectionDelayInSecs;
- private InetSocketAddress tsoAddr;
- private String zkCurrentTsoPath;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Construction
- // ----------------------------------------------------------------------------------------------------------------
-
- public static TSOClient newInstance(OmidClientConfiguration tsoClientConf)
- throws IOException, InterruptedException {
- return new TSOClient(tsoClientConf);
- }
-
- // Avoid instantiation
- private TSOClient(OmidClientConfiguration omidConf) throws IOException, InterruptedException {
-
- // Start client with Nb of active threads = 3 as maximum.
- int tsoExecutorThreads = omidConf.getExecutorThreads();
-
- factory = new NioClientSocketChannelFactory(
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), tsoExecutorThreads);
- // Create the bootstrap
- bootstrap = new ClientBootstrap(factory);
-
- requestTimeoutInMs = omidConf.getRequestTimeoutInMs();
- requestMaxRetries = omidConf.getRequestMaxRetries();
- tsoReconnectionDelayInSecs = omidConf.getReconnectionDelayInSecs();
-
- LOG.info("Connecting to TSO...");
- HostAndPort hp;
- switch (omidConf.getConnectionType()) {
- case HA:
- zkClient = ZKUtils.initZKClient(omidConf.getConnectionString(),
- omidConf.getZkNamespace(),
- omidConf.getZkConnectionTimeoutInSecs());
- zkCurrentTsoPath = omidConf.getZkCurrentTsoPath();
- configureCurrentTSOServerZNodeCache(zkCurrentTsoPath);
- String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
- // TSO info includes the new TSO host:port address and epoch
- String[] currentTSOAndEpochArray = tsoInfo.split("#");
- hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
- setTSOAddress(hp.getHostText(), hp.getPort());
- epoch = Long.parseLong(currentTSOAndEpochArray[1]);
- LOG.info("\t* Current TSO host:port found in ZK: {} Epoch {}", hp, getEpoch());
- break;
- case DIRECT:
- default:
- hp = HostAndPort.fromString(omidConf.getConnectionString());
- setTSOAddress(hp.getHostText(), hp.getPort());
- LOG.info("\t* TSO host:port {} will be connected directly", hp);
- break;
- }
-
- fsmExecutor = Executors.newSingleThreadScheduledExecutor(
- new ThreadFactoryBuilder().setNameFormat("tsofsm-%d").build());
- fsm = new StateMachine.FsmImpl(fsmExecutor);
- fsm.setInitState(new DisconnectedState(fsm));
-
- ChannelPipeline pipeline = bootstrap.getPipeline();
- pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
- pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
- pipeline.addLast("protobufencoder", new ProtobufEncoder());
- pipeline.addLast("handler", new Handler(fsm));
-
- bootstrap.setOption("tcpNoDelay", true);
- bootstrap.setOption("keepAlive", true);
- bootstrap.setOption("reuseAddress", true);
- bootstrap.setOption("connectTimeoutMillis", 100);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // TSOProtocol interface
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * @see TSOProtocol#getNewStartTimestamp()
- */
- @Override
- public TSOFuture<Long> getNewStartTimestamp() {
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.TimestampRequest.Builder tsreqBuilder = TSOProto.TimestampRequest.newBuilder();
- builder.setTimestampRequest(tsreqBuilder.build());
- RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
- fsm.sendEvent(request);
- return new ForwardingTSOFuture<>(request);
- }
-
- /**
- * @see TSOProtocol#commit(long, Set)
- */
- @Override
- public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitbuilder = TSOProto.CommitRequest.newBuilder();
- commitbuilder.setStartTimestamp(transactionId);
- for (CellId cell : cells) {
- commitbuilder.addCellId(cell.getCellId());
- }
- builder.setCommitRequest(commitbuilder.build());
- RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
- fsm.sendEvent(request);
- return new ForwardingTSOFuture<>(request);
- }
-
- /**
- * @see TSOProtocol#close()
- */
- @Override
- public TSOFuture<Void> close() {
- final CloseEvent closeEvent = new CloseEvent();
- fsm.sendEvent(closeEvent);
- closeEvent.addListener(new Runnable() {
- @Override
- public void run() {
- try {
- closeEvent.get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- e.printStackTrace();
- } catch (ExecutionException e) {
- e.printStackTrace();
- } finally {
- fsmExecutor.shutdown();
- if (currentTSOZNode != null) {
- try {
- currentTSOZNode.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- if (zkClient != null) {
- zkClient.close();
- }
- }
-
- }
- }, fsmExecutor);
- return new ForwardingTSOFuture<>(closeEvent);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // High availability related interface
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * Returns the epoch of the TSO server that initialized this transaction.
- * Used for high availability support.
- */
- public long getEpoch() {
- return epoch;
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // NodeCacheListener interface
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public void nodeChanged() throws Exception {
-
- String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
- // TSO info includes the new TSO host:port address and epoch
- String[] currentTSOAndEpochArray = tsoInfo.split("#");
- HostAndPort hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
- setTSOAddress(hp.getHostText(), hp.getPort());
- epoch = Long.parseLong(currentTSOAndEpochArray[1]);
- LOG.info("CurrentTSO ZNode changed. New TSO Host & Port {}/Epoch {}", hp, getEpoch());
- if (currentChannel != null && currentChannel.isConnected()) {
- LOG.info("\tClosing channel with previous TSO {}", currentChannel);
- currentChannel.close();
- }
-
- }
-
- // ****************************************** Finite State Machine ************************************************
-
- // ----------------------------------------------------------------------------------------------------------------
- // FSM: Events
- // ----------------------------------------------------------------------------------------------------------------
-
- private static class ParamEvent<T> implements StateMachine.Event {
-
- final T param;
-
- ParamEvent(T param) {
- this.param = param;
- }
-
- T getParam() {
- return param;
- }
- }
-
- private static class ErrorEvent extends ParamEvent<Throwable> {
-
- ErrorEvent(Throwable t) {
- super(t);
- }
- }
-
- private static class ConnectedEvent extends ParamEvent<Channel> {
-
- ConnectedEvent(Channel c) {
- super(c);
- }
- }
-
- private static class UserEvent<T> extends AbstractFuture<T>
- implements StateMachine.DeferrableEvent {
-
- void success(T value) {
- set(value);
- }
-
- @Override
- public void error(Throwable t) {
- setException(t);
- }
- }
-
- private static class CloseEvent extends UserEvent<Void> {
-
- }
-
- private static class ChannelClosedEvent extends ParamEvent<Throwable> {
-
- ChannelClosedEvent(Throwable t) {
- super(t);
- }
- }
-
- private static class ReconnectEvent implements StateMachine.Event {
-
- }
-
- private static class HandshakeTimeoutEvent implements StateMachine.Event {
-
- }
-
- private static class TimestampRequestTimeoutEvent implements StateMachine.Event {
-
- }
-
- private static class CommitRequestTimeoutEvent implements StateMachine.Event {
-
- final long startTimestamp;
-
- CommitRequestTimeoutEvent(long startTimestamp) {
- this.startTimestamp = startTimestamp;
- }
-
- public long getStartTimestamp() {
- return startTimestamp;
- }
- }
-
- private static class RequestEvent extends UserEvent<Long> {
-
- TSOProto.Request req;
- int retriesLeft;
-
- RequestEvent(TSOProto.Request req, int retriesLeft) {
- this.req = req;
- this.retriesLeft = retriesLeft;
- }
-
- TSOProto.Request getRequest() {
- return req;
- }
-
- void setRequest(TSOProto.Request request) {
- this.req = request;
- }
-
- int getRetriesLeft() {
- return retriesLeft;
- }
-
- void decrementRetries() {
- retriesLeft--;
- }
-
- }
-
- private static class ResponseEvent extends ParamEvent<TSOProto.Response> {
-
- ResponseEvent(TSOProto.Response r) {
- super(r);
- }
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // FSM: States
- // ----------------------------------------------------------------------------------------------------------------
-
- class BaseState extends StateMachine.State {
-
- BaseState(StateMachine.Fsm fsm) {
- super(fsm);
- }
-
- public StateMachine.State handleEvent(StateMachine.Event e) {
- LOG.error("Unhandled event {} while in state {}", e, this.getClass().getName());
- return this;
- }
- }
-
- class DisconnectedState extends BaseState {
-
- DisconnectedState(StateMachine.Fsm fsm) {
- super(fsm);
- LOG.debug("NEW STATE: DISCONNECTED");
- }
-
- public StateMachine.State handleEvent(RequestEvent e) {
- fsm.deferEvent(e);
- return tryToConnectToTSOServer();
- }
-
- public StateMachine.State handleEvent(CloseEvent e) {
- factory.releaseExternalResources();
- e.success(null);
- return this;
- }
-
- private StateMachine.State tryToConnectToTSOServer() {
- final InetSocketAddress tsoAddress = getAddress();
- LOG.info("Trying to connect to TSO [{}]", tsoAddress);
- ChannelFuture channelFuture = bootstrap.connect(tsoAddress);
- channelFuture.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture channelFuture) throws Exception {
- if (channelFuture.isSuccess()) {
- LOG.info("Connection to TSO [{}] established. Channel {}",
- tsoAddress, channelFuture.getChannel());
- } else {
- LOG.error("Failed connection attempt to TSO [{}] failed. Channel {}",
- tsoAddress, channelFuture.getChannel());
- }
- }
- });
- return new ConnectingState(fsm);
- }
- }
-
- private class ConnectingState extends BaseState {
-
- ConnectingState(StateMachine.Fsm fsm) {
- super(fsm);
- LOG.debug("NEW STATE: CONNECTING");
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- fsm.deferEvent(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ConnectedEvent e) {
- return new HandshakingState(fsm, e.getParam());
- }
-
- public StateMachine.State handleEvent(ChannelClosedEvent e) {
- return new ConnectionFailedState(fsm, e.getParam());
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- return new ConnectionFailedState(fsm, e.getParam());
- }
-
- }
-
- private static class RequestAndTimeout {
-
- final RequestEvent event;
- final Timeout timeout;
-
- RequestAndTimeout(RequestEvent event, Timeout timeout) {
- this.event = event;
- this.timeout = timeout;
- }
-
- RequestEvent getRequest() {
- return event;
- }
-
- Timeout getTimeout() {
- return timeout;
- }
-
- public String toString() {
- String info = "Request type ";
- if (event.getRequest().hasTimestampRequest()) {
- info += "[Timestamp]";
- } else if (event.getRequest().hasCommitRequest()) {
- info += "[Commit] Start TS ->" + event.getRequest().getCommitRequest().getStartTimestamp();
- } else {
- info += "NONE";
- }
- return info;
- }
- }
-
- private class HandshakingState extends BaseState {
-
- final Channel channel;
-
- final HashedWheelTimer timeoutExecutor = new HashedWheelTimer(
- new ThreadFactoryBuilder().setNameFormat("tso-client-timeout").build());
- final Timeout timeout;
-
- HandshakingState(StateMachine.Fsm fsm, Channel channel) {
- super(fsm);
- LOG.debug("NEW STATE: HANDSHAKING");
- this.channel = channel;
- TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
- // Add the required handshake capabilities when necessary
- handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
- channel.write(TSOProto.Request.newBuilder().setHandshakeRequest(handshake.build()).build());
- timeout = newTimeout();
- }
-
- private Timeout newTimeout() {
- if (requestTimeoutInMs > 0) {
- return timeoutExecutor.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) {
- fsm.sendEvent(new HandshakeTimeoutEvent());
- }
- }, 30, TimeUnit.SECONDS);
- } else {
- return null;
- }
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- fsm.deferEvent(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ResponseEvent e) {
- if (e.getParam().hasHandshakeResponse() && e.getParam().getHandshakeResponse().getClientCompatible()) {
- if (timeout != null) {
- timeout.cancel();
- }
- return new ConnectedState(fsm, channel, timeoutExecutor);
- } else {
- cleanupState();
- LOG.error("Client incompatible with server");
- return new HandshakeFailedState(fsm, new HandshakeFailedException());
- }
- }
-
- public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
- cleanupState();
- return new ClosingState(fsm);
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- cleanupState();
- Throwable exception = e.getParam();
- LOG.error("Error during handshake", exception);
- return new HandshakeFailedState(fsm, exception);
- }
-
- private void cleanupState() {
- timeoutExecutor.stop();
- channel.close();
- if (timeout != null) {
- timeout.cancel();
- }
- }
-
- }
-
- class ConnectionFailedState extends BaseState {
-
- final HashedWheelTimer reconnectionTimeoutExecutor = new HashedWheelTimer(
- new ThreadFactoryBuilder().setNameFormat("tso-client-backoff-timeout").build());
-
- Throwable exception;
-
- ConnectionFailedState(final StateMachine.Fsm fsm, final Throwable exception) {
- super(fsm);
- LOG.debug("NEW STATE: CONNECTION FAILED [RE-CONNECTION BACKOFF]");
- this.exception = exception;
- reconnectionTimeoutExecutor.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) {
- fsm.sendEvent(new ReconnectEvent());
- }
- }, tsoReconnectionDelayInSecs, TimeUnit.SECONDS);
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- e.error(exception);
- return this;
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- return this;
- }
-
- public StateMachine.State handleEvent(ChannelClosedEvent e) {
- return new DisconnectedState(fsm);
- }
-
- public StateMachine.State handleEvent(ReconnectEvent e) {
- return new DisconnectedState(fsm);
- }
-
- }
-
- private class HandshakeFailedState extends ConnectionFailedState {
-
- HandshakeFailedState(StateMachine.Fsm fsm, Throwable exception) {
- super(fsm, exception);
- LOG.debug("STATE: HANDSHAKING FAILED");
- }
-
- }
-
- class ConnectedState extends BaseState {
-
- final Queue<RequestAndTimeout> timestampRequests;
- final Map<Long, RequestAndTimeout> commitRequests;
- final Channel channel;
-
- final HashedWheelTimer timeoutExecutor;
-
- ConnectedState(StateMachine.Fsm fsm, Channel channel, HashedWheelTimer timeoutExecutor) {
- super(fsm);
- LOG.debug("NEW STATE: CONNECTED");
- this.channel = channel;
- this.timeoutExecutor = timeoutExecutor;
- timestampRequests = new ArrayDeque<>();
- commitRequests = new HashMap<>();
- }
-
- private Timeout newTimeout(final StateMachine.Event timeoutEvent) {
- if (requestTimeoutInMs > 0) {
- return timeoutExecutor.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) {
- fsm.sendEvent(timeoutEvent);
- }
- }, requestTimeoutInMs, TimeUnit.MILLISECONDS);
- } else {
- return null;
- }
- }
-
- private void sendRequest(final StateMachine.Fsm fsm, RequestEvent request) {
- TSOProto.Request req = request.getRequest();
-
- if (req.hasTimestampRequest()) {
- timestampRequests.add(new RequestAndTimeout(request, newTimeout(new TimestampRequestTimeoutEvent())));
- } else if (req.hasCommitRequest()) {
- TSOProto.CommitRequest commitReq = req.getCommitRequest();
- commitRequests.put(commitReq.getStartTimestamp(), new RequestAndTimeout(
- request, newTimeout(new CommitRequestTimeoutEvent(commitReq.getStartTimestamp()))));
- } else {
- request.error(new IllegalArgumentException("Unknown request type"));
- return;
- }
- ChannelFuture f = channel.write(req);
-
- f.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture future) {
- if (!future.isSuccess()) {
- fsm.sendEvent(new ErrorEvent(future.getCause()));
- }
- }
- });
- }
-
- private void handleResponse(ResponseEvent response) {
- TSOProto.Response resp = response.getParam();
- if (resp.hasTimestampResponse()) {
- if (timestampRequests.size() == 0) {
- LOG.debug("Received timestamp response when no requests outstanding");
- return;
- }
- RequestAndTimeout e = timestampRequests.remove();
- e.getRequest().success(resp.getTimestampResponse().getStartTimestamp());
- if (e.getTimeout() != null) {
- e.getTimeout().cancel();
- }
- } else if (resp.hasCommitResponse()) {
- long startTimestamp = resp.getCommitResponse().getStartTimestamp();
- RequestAndTimeout e = commitRequests.remove(startTimestamp);
- if (e == null) {
- LOG.debug("Received commit response for request that doesn't exist. Start TS: {}", startTimestamp);
- return;
- }
- if (e.getTimeout() != null) {
- e.getTimeout().cancel();
- }
- if (resp.getCommitResponse().getAborted()) {
- e.getRequest().error(new AbortException());
- } else {
- // Check if the commit response received implies heuristic
- // actions during commit (because there's a new TSO master
- // replica) and inform the caller (e.g. the TxMgr) about it
- if (resp.getCommitResponse().getMakeHeuristicDecision()) {
- e.getRequest().error(new NewTSOException());
- } else {
- e.getRequest().success(resp.getCommitResponse().getCommitTimestamp());
- }
- }
- }
- }
-
- public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
- if (!timestampRequests.isEmpty()) {
- RequestAndTimeout r = timestampRequests.remove();
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- }
- return this;
- }
-
- public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
- long startTimestamp = e.getStartTimestamp();
- if (commitRequests.containsKey(startTimestamp)) {
- RequestAndTimeout r = commitRequests.remove(startTimestamp);
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- }
- return this;
- }
-
- public StateMachine.State handleEvent(CloseEvent e) {
- LOG.debug("CONNECTED STATE: CloseEvent");
- timeoutExecutor.stop();
- closeChannelAndErrorRequests();
- fsm.deferEvent(e);
- return new ClosingState(fsm);
- }
-
- public StateMachine.State handleEvent(RequestEvent e) {
- sendRequest(fsm, e);
- return this;
- }
-
- public StateMachine.State handleEvent(ResponseEvent e) {
- handleResponse(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- LOG.debug("CONNECTED STATE: ErrorEvent");
- timeoutExecutor.stop();
- handleError(fsm);
- return new ClosingState(fsm);
- }
-
- private void handleError(StateMachine.Fsm fsm) {
- LOG.debug("CONNECTED STATE: Cancelling Timeouts in handleError");
- while (timestampRequests.size() > 0) {
- RequestAndTimeout r = timestampRequests.remove();
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- }
- Iterator<Map.Entry<Long, RequestAndTimeout>> iter = commitRequests.entrySet().iterator();
- while (iter.hasNext()) {
- RequestAndTimeout r = iter.next().getValue();
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- iter.remove();
- }
- channel.close();
- }
-
- private void queueRetryOrError(StateMachine.Fsm fsm, RequestEvent e) {
- if (e.getRetriesLeft() > 0) {
- e.decrementRetries();
- if (e.getRequest().hasCommitRequest()) {
- TSOProto.CommitRequest commitRequest = e.getRequest().getCommitRequest();
- if (!commitRequest.getIsRetry()) { // Create a new retry for the commit request
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
- commitBuilder.mergeFrom(commitRequest);
- commitBuilder.setIsRetry(true);
- builder.setCommitRequest(commitBuilder.build());
- e.setRequest(builder.build());
- }
- }
- fsm.sendEvent(e);
- } else {
- e.error(
- new ServiceUnavailableException("Number of retries exceeded. This API request failed permanently"));
- }
- }
-
- private void closeChannelAndErrorRequests() {
- channel.close();
- for (RequestAndTimeout r : timestampRequests) {
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- r.getRequest().error(new ClosingException());
- }
- for (RequestAndTimeout r : commitRequests.values()) {
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- r.getRequest().error(new ClosingException());
- }
- }
- }
-
- private class ClosingState extends BaseState {
-
- ClosingState(StateMachine.Fsm fsm) {
- super(fsm);
- LOG.debug("NEW STATE: CLOSING");
- }
-
- public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(ResponseEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- fsm.deferEvent(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ChannelClosedEvent e) {
- return new DisconnectedState(fsm);
- }
-
- public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
- return this;
- }
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper classes & methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private class Handler extends SimpleChannelHandler {
-
- private StateMachine.Fsm fsm;
-
- Handler(StateMachine.Fsm fsm) {
- this.fsm = fsm;
- }
-
- @Override
- public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
- currentChannel = e.getChannel();
- LOG.debug("HANDLER (CHANNEL CONNECTED): Connection {}. Sending connected event to FSM", e);
- fsm.sendEvent(new ConnectedEvent(e.getChannel()));
- }
-
- @Override
- public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- LOG.debug("HANDLER (CHANNEL DISCONNECTED): Connection {}. Sending error event to FSM", e);
- fsm.sendEvent(new ErrorEvent(new ConnectionException()));
- }
-
- @Override
- public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- LOG.debug("HANDLER (CHANNEL CLOSED): Connection {}. Sending channel closed event to FSM", e);
- fsm.sendEvent(new ChannelClosedEvent(new ConnectionException()));
- }
-
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
- if (e.getMessage() instanceof TSOProto.Response) {
- fsm.sendEvent(new ResponseEvent((TSOProto.Response) e.getMessage()));
- } else {
- LOG.warn("Received unknown message", e.getMessage());
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
- LOG.error("Error on channel {}", ctx.getChannel(), e.getCause());
- fsm.sendEvent(new ErrorEvent(e.getCause()));
- }
- }
-
- private synchronized void setTSOAddress(String host, int port) {
- tsoAddr = new InetSocketAddress(host, port);
- }
-
- private synchronized InetSocketAddress getAddress() {
- return tsoAddr;
- }
-
- private void configureCurrentTSOServerZNodeCache(String currentTsoPath) {
- try {
- currentTSOZNode = new NodeCache(zkClient, currentTsoPath);
- currentTSOZNode.getListenable().addListener(this);
- currentTSOZNode.start(true);
- } catch (Exception e) {
- throw new IllegalStateException("Cannot start watcher on current TSO Server ZNode: " + e.getMessage());
- }
- }
-
- private String getCurrentTSOInfoFoundInZK(String currentTsoPath) {
- ChildData currentTSOData = currentTSOZNode.getCurrentData();
- if (currentTSOData == null) {
- throw new IllegalStateException("No data found in ZKNode " + currentTsoPath);
- }
- byte[] currentTSOAndEpochAsBytes = currentTSOData.getData();
- if (currentTSOAndEpochAsBytes == null) {
- throw new IllegalStateException("No data found for current TSO in ZKNode " + currentTsoPath);
- }
- return new String(currentTSOAndEpochAsBytes, Charsets.UTF_8);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java
deleted file mode 100644
index 8f42353..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import java.util.concurrent.Executor;
-import java.util.concurrent.Future;
-
-public interface TSOFuture<T> extends Future<T> {
- public void addListener(Runnable listener, Executor executor);
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java
deleted file mode 100644
index cb9e679..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import java.util.Set;
-
-/**
- * Defines the protocol used on the client side to abstract communication to the TSO server
- */
-public interface TSOProtocol {
-
- /**
- * Returns a new timestamp assigned by on the server-side
- * @return the newly assigned timestamp as a future. If an error was detected, the future will contain a
- * corresponding protocol exception
- * @see TimestampOracle
- * @see TSOServer
- */
- TSOFuture<Long> getNewStartTimestamp();
-
- /**
- * Returns the result of the conflict detection made on the server-side for the specified transaction
- * @param transactionId
- * the transaction to check for conflicts
- * @param writeSet
- * the writeSet of the transaction, which includes all the modified cells
- * @return the commit timestamp as a future if the transaction was committed. If the transaction was aborted due
- * to conflicts with a concurrent transaction, the future will include an AbortException. If an error was detected,
- * the future will contain a corresponding protocol exception
- * @see TimestampOracle
- * @see TSOServer
- */
- TSOFuture<Long> commit(long transactionId, Set<? extends CellId> writeSet);
-
- /**
- * Closes the communication with the TSO server
- * @return nothing. If an error was detected, the future will contain a corresponding protocol exception
- */
- TSOFuture<Void> close();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java b/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
deleted file mode 100644
index d4421e0..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.util;
-
-import com.yahoo.omid.tso.client.CellId;
-
-public class DummyCellIdImpl implements CellId {
-
- private final long cellId;
-
- public DummyCellIdImpl(long cellId) {
- this.cellId = cellId;
- }
-
- @Override
- public long getCellId() {
- return cellId;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransaction.java b/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransaction.java
new file mode 100644
index 0000000..e559609
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransaction.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+import org.apache.omid.tso.client.CellId;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Omid's base abstract implementation of the {@link Transaction} interface.
+ * Provides extra methods to access other basic transaction state required by
+ * {@link TransactionManagerExtension} implementations based on snapshot
+ * isolation.
+ *
+ * So, this abstract class must be extended by particular implementations of
+ * transaction managers related to different storage systems (HBase...)
+ */
+public abstract class AbstractTransaction<T extends CellId> implements Transaction {
+
+ private transient Map<String, Object> metadata = new HashMap<>();
+ private final AbstractTransactionManager transactionManager;
+ private final long startTimestamp;
+ private final long epoch;
+ private long commitTimestamp;
+ private boolean isRollbackOnly;
+ private final Set<T> writeSet;
+ private Status status = Status.RUNNING;
+
+ /**
+ * Base constructor
+ *
+ * @param transactionId
+ * transaction identifier to assign
+ * @param epoch
+ * epoch of the TSOServer instance that created this transaction
+ * Used in High Availability to guarantee data consistency
+ * @param writeSet
+ * initial write set for the transaction.
+ * Should be empty in most cases.
+ * @param transactionManager
+ * transaction manager associated to this transaction.
+ * Usually, should be the one that created the transaction
+ * instance.
+ */
+ public AbstractTransaction(long transactionId,
+ long epoch,
+ Set<T> writeSet,
+ AbstractTransactionManager transactionManager) {
+ this.startTimestamp = transactionId;
+ this.epoch = epoch;
+ this.writeSet = writeSet;
+ this.transactionManager = transactionManager;
+ }
+
+ /**
+ * Allows to define specific clean-up task for transaction implementations
+ */
+ public abstract void cleanup();
+
+ /**
+ * @see org.apache.omid.transaction.Transaction#getTransactionId()
+ */
+ @Override
+ public long getTransactionId() {
+ return startTimestamp;
+ }
+
+ /**
+ * @see org.apache.omid.transaction.Transaction#getEpoch()
+ */
+ @Override
+ public long getEpoch() {
+ return epoch;
+ }
+
+ /**
+ * @see org.apache.omid.transaction.Transaction#getStatus()
+ */
+ @Override
+ public Status getStatus() {
+ return status;
+ }
+
+ /**
+ * @see Transaction#isRollbackOnly()
+ */
+ @Override
+ public void setRollbackOnly() {
+ isRollbackOnly = true;
+ }
+
+ /**
+ * @see org.apache.omid.transaction.Transaction#isRollbackOnly()
+ */
+ @Override
+ public boolean isRollbackOnly() {
+ return isRollbackOnly;
+ }
+
+ /**
+ * Returns transaction manager associated to this transaction.
+ * @return transaction manager
+ */
+ public AbstractTransactionManager getTransactionManager() {
+ return transactionManager;
+ }
+
+ /**
+ * Returns the start timestamp for this transaction.
+ * @return start timestamp
+ */
+ public long getStartTimestamp() {
+ return startTimestamp;
+ }
+
+ /**
+ * Returns the commit timestamp for this transaction.
+ * @return commit timestamp
+ */
+ public long getCommitTimestamp() {
+ return commitTimestamp;
+ }
+
+ /**
+ * Sets the commit timestamp for this transaction.
+ * @param commitTimestamp
+ * the commit timestamp to set
+ */
+ public void setCommitTimestamp(long commitTimestamp) {
+ this.commitTimestamp = commitTimestamp;
+ }
+
+ /**
+ * Sets the status for this transaction.
+ * @param status
+ * the {@link Status} to set
+ */
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+
+ /**
+ * Returns the current write-set for this transaction.
+ * @return write set
+ */
+ public Set<T> getWriteSet() {
+ return writeSet;
+ }
+
+ /**
+ * Adds an element to the transaction write-set.
+ * @param element
+ * the element to add
+ */
+ public void addWriteSetElement(T element) {
+ writeSet.add(element);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("Tx-%s [%s] (ST=%d, CT=%d, Epoch=%d) WriteSet %s",
+ Long.toHexString(getTransactionId()),
+ status,
+ startTimestamp,
+ commitTimestamp,
+ epoch,
+ writeSet);
+ }
+
+ @Override
+ public Optional<Object> getMetadata(String key) {
+ return Optional.fromNullable(metadata.get(key));
+ }
+
+ /**
+ * Expects they metadata stored under key "key" to be of the "Set" type,
+ * append "value" to the existing set or creates a new one
+ */
+ @Override
+ @SuppressWarnings("unchecked")
+ public void appendMetadata(String key, Object value) {
+ List existingValue = (List) metadata.get(key);
+ if (existingValue == null) {
+ List<Object> newList = new ArrayList<>();
+ newList.add(value);
+ metadata.put(key, newList);
+ } else {
+ existingValue.add(value);
+ }
+ }
+
+ @Override
+ public void setMetadata(String key, Object value) {
+ metadata.put(key, value);
+ }
+
+}
[07/52] [abbrv] incubator-omid git commit: [ci skip]prepare for next
development iteration
Posted by ik...@apache.org.
[ci skip]prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/516e517d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/516e517d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/516e517d
Branch: refs/heads/master
Commit: 516e517d9dab274f9ec758af450e730e7ac27df3
Parents: a3887f2
Author: Omid CI <om...@yahoo-inc.com>
Authored: Wed Apr 13 21:10:10 2016 +0000
Committer: Omid CI <om...@yahoo-inc.com>
Committed: Wed Apr 13 21:10:10 2016 +0000
----------------------------------------------------------------------
benchmarks/pom.xml | 2 +-
codahale-metrics/pom.xml | 2 +-
commit-table/pom.xml | 2 +-
common/pom.xml | 2 +-
examples/pom.xml | 2 +-
hbase-client/pom.xml | 2 +-
hbase-commit-table/pom.xml | 2 +-
hbase-common/pom.xml | 2 +-
hbase-coprocessor/pom.xml | 2 +-
hbase-shims/hbase-0/pom.xml | 2 +-
hbase-shims/hbase-1/pom.xml | 2 +-
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 2 +-
metrics/pom.xml | 2 +-
pom.xml | 4 ++--
statemachine/pom.xml | 2 +-
timestamp-storage/pom.xml | 2 +-
transaction-client/pom.xml | 2 +-
tso-server/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index d5a3c72..d2366f4 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>benchmarks</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index 362ae0d..510778c 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index b069d15..872dda8 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 8ae6dc9..67ab4ef 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index d0b027b..da4d918 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 4238091..9b5a7ca 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index 63e2a76..3fe214d 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase-commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 2adfd62..2c307f5 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index f90e4cc..6781a50 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase-coprocessor</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index 5be179d..8f1f530 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase0-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index 4e556ac..7539e77 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase1-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index 8fc98e5..e1ec152 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>omid-shims-aggregator</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index 6aba19f..e1d6ab8 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>hbase-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index e481c73..dd5c642 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>metrics</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 402423e..b3b6af5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
<licenses>
<license>
@@ -46,7 +46,7 @@
<connection>scm:git:git://github.com/yahoo/omid.git</connection>
<developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
<url>https://github.com/yahoo/omid</url>
- <tag>omid-0.8.1.35</tag>
+ <tag>omid-0.8.1.21</tag>
</scm>
<distributionManagement>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index 047c265..717886f 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<name>State Machine</name>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index a512f12..6085da0 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>timestamp-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index f398046..2531c5d 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>transaction-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/516e517d/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index 22a7e3b..32d6cef 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.35</version>
+ <version>0.8.1.36-SNAPSHOT</version>
</parent>
<artifactId>tso-server</artifactId>
[03/52] [abbrv] incubator-omid git commit: Updated source code with
license header
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
index 6f720dc..8394a01 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.client.Get;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
index 81faa91..660ed78 100644
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
+++ b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
index 71db3df..57efffd 100644
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
+++ b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
index e3f9387..a0687a1 100644
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
+++ b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
index 7e73c0c..4c0d617 100644
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
+++ b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java b/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
index e1a75cf..a460c76 100644
--- a/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
+++ b/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.committable.hbase;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java b/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
index 394b181..2b51777 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
index dccafc1..55a12dd 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
index f0423ee..2287afb 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGeneratorImplementations.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java b/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
index bf97c81..b4fe81d 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/HBaseLogin.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java b/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
index db2f67b..8214780 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/tools/hbase/SecureHBaseConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java b/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
index 82af3d0..7dd0988 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/transaction/CellInfo.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java b/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
index 8f06e6e..0d599f6 100644
--- a/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
+++ b/hbase-common/src/main/java/com/yahoo/omid/transaction/CellUtils.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java b/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
index 10a583b..674cdd7 100644
--- a/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
+++ b/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.committable.hbase;
import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.BadRandomKeyGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
index 51cf10f..e391b2b 100644
--- a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
+++ b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
index 1680923..e3e181e 100644
--- a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
+++ b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
index a74dec5..8ab18b6 100644
--- a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
+++ b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java b/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
index e794def..17075f9 100644
--- a/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
+++ b/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
index a1e52d9..6d0f519 100644
--- a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
+++ b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.inject.AbstractModule;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
index ac1ae78..4fe440e 100644
--- a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
+++ b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompaction.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.SettableFuture;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
index 84edfa5..d157512 100644
--- a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
+++ b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TestCompactorScanner.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.SettableFuture;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-coprocessor/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/resources/log4j.xml b/hbase-coprocessor/src/test/resources/log4j.xml
index 6342036..225b90c 100755
--- a/hbase-coprocessor/src/test/resources/log4j.xml
+++ b/hbase-coprocessor/src/test/resources/log4j.xml
@@ -1,4 +1,23 @@
<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+-->
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j='http://jakarta.apache.org/log4j/'>
<appender name="console" class="org.apache.log4j.ConsoleAppender">
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java b/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
index da28809..fb3c690 100644
--- a/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
+++ b/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java b/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
index 84e1e4f..81db563 100644
--- a/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
+++ b/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/Region.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java b/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
index 3b4caa5..2be33e1 100644
--- a/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
+++ b/hbase-shims/hbase-0/src/main/java/org/apache/hadoop/hbase/regionserver/ScannerContext.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java b/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
index c5eda5a..6e54cee 100644
--- a/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
+++ b/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
----------------------------------------------------------------------
diff --git a/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java b/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
index 1244bc9..2577e40 100644
--- a/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
+++ b/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
----------------------------------------------------------------------
diff --git a/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java b/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
index 645679b..7773eeb 100644
--- a/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
+++ b/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tools.hbase;
import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java b/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
index 34e290b..d8d340a 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java b/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
index fac88d7..11de43c 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java b/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
index 6bd4cd2..3084803 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java b/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
index 23ea57b..a8c07cb 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java b/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
index b859e03..9a582f9 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java b/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
index a750868..733c05f 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
index d34cb71..515a3da 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
index bde467b..16c49f1 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
index 67bcecb..a96fc74 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
index 6a99a51..1fb2d8c 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java b/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
index b5e1adb..01c906a 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java b/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
index f09b931..8a22582 100644
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
+++ b/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/misc/header.txt
----------------------------------------------------------------------
diff --git a/misc/header.txt b/misc/header.txt
index 0c3812a..f735fc6 100644
--- a/misc/header.txt
+++ b/misc/header.txt
@@ -1,10 +1,12 @@
-Copyright ${inceptionYear}-${latestYearOfContribution} ${owner}
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
+ http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 184eb05..10e4438 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,9 +105,7 @@
<!-- Licensing properties (for license-maven-plugins) -->
<license.header>misc/header.txt</license.header>
- <license.owner>Yahoo Inc.</license.owner>
- <license.inceptionYear>${project.inceptionYear}</license.inceptionYear>
- <license.currentYear>2016</license.currentYear> <!-- Manually assigned -->
+
</properties>
@@ -286,11 +284,6 @@
<header>${license.header}</header>
<aggregate>true</aggregate> <!-- for multi-module config -->
<skipExistingHeaders>false</skipExistingHeaders>
- <properties>
- <owner>${license.owner}</owner>
- <inceptionYear>${project.inceptionYear}</inceptionYear>
- <latestYearOfContribution>${license.currentYear}</latestYearOfContribution>
- </properties>
<includes>
<!-- Add files where to include licensing -->
<include>src/**/*.java</include>
@@ -312,9 +305,6 @@
<!-- Exclude assembly -->
<exclude>**/maven/assembly/*</exclude>
- <!-- Exclude tests -->
- <exclude>**/src/test/**/*</exclude>
-
<!-- Project specific exclusions (e.g. they've been copied/modified from other open-source projects -->
<!-- Protocol buffers generated classes -->
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
----------------------------------------------------------------------
diff --git a/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java b/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
index 88bd86f..a1ff834 100644
--- a/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
+++ b/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
----------------------------------------------------------------------
diff --git a/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java b/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
index 8541c77..7b66d8a 100644
--- a/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
+++ b/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java b/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
index 641c87c..9f90163 100644
--- a/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
+++ b/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.statemachine;
import com.yahoo.statemachine.StateMachine.Fsm;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
index 07267bd..4d84cc8 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
index b9624c8..525a2d7 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
index 823bded..f56aea4 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
index d613353..829201b 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
index bffccaf..1246e43 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
index a72466b..00c92b7 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
index 566431c..57dfe71 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
index 53a058d..88d0ea0 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
index d7d8985..173b9fa 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
index af66183..02be0ea 100644
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
+++ b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java b/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
index bfa0847..c273995 100644
--- a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
+++ b/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.timestamp.storage;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java b/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
index a7fec99..363768f 100644
--- a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
+++ b/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.timestamp.storage;
import org.apache.curator.RetryPolicy;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
index 7abec48..d303a04 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
index 1ee91bc..83c766a 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
[33/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
deleted file mode 100644
index 660ed78..0000000
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.name.Names;
-import com.yahoo.omid.HBaseConfigModule;
-
-/**
- * This class is instantiated by the yaml parser.
- * Snake_yaml needs a public POJO style class to work properly with all the setters and getters.
- */
-public class DefaultHBaseCommitTableStorageModule extends AbstractModule {
-
- private String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
- private String familyName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME;
- private String lowWatermarkFamily = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME;
- private String keytab;
- private String principal;
-
- @Override
- protected void configure() {
- bindConstant().annotatedWith(Names.named(HBaseCommitTableConfig.COMMIT_TABLE_NAME_KEY)).to(tableName);
- bindConstant().annotatedWith(Names.named(HBaseCommitTableConfig.COMMIT_TABLE_CF_NAME_KEY)).to(familyName);
- bindConstant().annotatedWith(Names.named(HBaseCommitTableConfig.COMMIT_TABLE_LWM_CF_NAME_KEY)).to(lowWatermarkFamily);
- install(new HBaseConfigModule(principal, keytab));
- install(new HBaseCommitTableStorageModule());
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // WARNING: Do not remove getters/setters, needed by snake_yaml!
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public String getFamilyName() {
- return familyName;
- }
-
- public void setFamilyName(String familyName) {
- this.familyName = familyName;
- }
-
- public String getLowWatermarkFamily() {
- return lowWatermarkFamily;
- }
-
- public void setLowWatermarkFamily(String lowWatermarkFamily) {
- this.lowWatermarkFamily = lowWatermarkFamily;
- }
-
- public String getPrincipal() {
- return principal;
- }
-
- public void setPrincipal(String principal) {
- this.principal = principal;
- }
-
- public String getKeytab() {
- return keytab;
- }
-
- public void setKeytab(String keytab) {
- this.keytab = keytab;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
deleted file mode 100644
index 57efffd..0000000
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTable.java
+++ /dev/null
@@ -1,422 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.AbstractFuture;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static com.yahoo.omid.committable.hbase.HBaseCommitTableConfig.COMMIT_TABLE_QUALIFIER;
-import static com.yahoo.omid.committable.hbase.HBaseCommitTableConfig.INVALID_TX_QUALIFIER;
-import static com.yahoo.omid.committable.hbase.HBaseCommitTableConfig.LOW_WATERMARK_QUALIFIER;
-import static com.yahoo.omid.committable.hbase.HBaseCommitTableConfig.LOW_WATERMARK_ROW;
-
-public class HBaseCommitTable implements CommitTable {
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseCommitTable.class);
-
- private final Configuration hbaseConfig;
- private final String tableName;
- private final byte[] commitTableFamily;
- private final byte[] lowWatermarkFamily;
- private final KeyGenerator keygen;
-
- /**
- * Create a hbase commit table.
- * Note that we do not take ownership of the passed htable, it is just used to construct the writer and client.
- */
- @Inject
- public HBaseCommitTable(Configuration hbaseConfig, HBaseCommitTableConfig config) {
- this(hbaseConfig, config, KeyGeneratorImplementations.defaultKeyGenerator());
- }
-
- public HBaseCommitTable(Configuration hbaseConfig, HBaseCommitTableConfig config, KeyGenerator keygen) {
-
- this.hbaseConfig = hbaseConfig;
- this.tableName = config.getTableName();
- this.commitTableFamily = config.getCommitTableFamily();
- this.lowWatermarkFamily = config.getLowWatermarkFamily();
- this.keygen = keygen;
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Reader and Writer
- // ----------------------------------------------------------------------------------------------------------------
-
- private class HBaseWriter implements Writer {
-
- private static final long INITIAL_LWM_VALUE = -1L;
- final HTable table;
- // Our own buffer for operations
- final List<Put> writeBuffer = new LinkedList<>();
- volatile long lowWatermarkToStore = INITIAL_LWM_VALUE;
-
- HBaseWriter() throws IOException {
- table = new HTable(hbaseConfig, tableName);
- }
-
- @Override
- public void addCommittedTransaction(long startTimestamp, long commitTimestamp) throws IOException {
- assert (startTimestamp < commitTimestamp);
- Put put = new Put(startTimestampToKey(startTimestamp), startTimestamp);
- byte[] value = encodeCommitTimestamp(startTimestamp, commitTimestamp);
- put.add(commitTableFamily, COMMIT_TABLE_QUALIFIER, value);
- writeBuffer.add(put);
- }
-
- @Override
- public void updateLowWatermark(long lowWatermark) throws IOException {
- lowWatermarkToStore = lowWatermark;
- }
-
- @Override
- public void flush() throws IOException {
- try {
- addLowWatermarkToStoreToWriteBuffer();
- table.put(writeBuffer);
- writeBuffer.clear();
- } catch (IOException e) {
- LOG.error("Error flushing data", e);
- throw e;
- }
- }
-
- @Override
- public void clearWriteBuffer() {
- writeBuffer.clear();
- }
-
- @Override
- public void close() throws IOException {
- clearWriteBuffer();
- table.close();
- }
-
- private void addLowWatermarkToStoreToWriteBuffer() {
- long lowWatermark = lowWatermarkToStore;
- if(lowWatermark != INITIAL_LWM_VALUE) {
- Put put = new Put(LOW_WATERMARK_ROW);
- put.add(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER, Bytes.toBytes(lowWatermark));
- writeBuffer.add(put);
- }
- }
-
- }
-
- class HBaseClient implements Client, Runnable {
-
- final HTable table;
- final HTable deleteTable;
- final ExecutorService deleteBatchExecutor;
- final BlockingQueue<DeleteRequest> deleteQueue;
- boolean isClosed = false; // @GuardedBy("this")
- final static int DELETE_BATCH_SIZE = 1024;
-
- HBaseClient() throws IOException {
- table = new HTable(hbaseConfig, tableName);
- table.setAutoFlush(false, true);
- deleteTable = new HTable(hbaseConfig, tableName);
- deleteQueue = new ArrayBlockingQueue<>(DELETE_BATCH_SIZE);
-
- deleteBatchExecutor = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("omid-completor-%d").build());
- deleteBatchExecutor.submit(this);
-
- }
-
- @Override
- public ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp) {
-
- SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
- try {
- Get get = new Get(startTimestampToKey(startTimestamp));
- get.addColumn(commitTableFamily, COMMIT_TABLE_QUALIFIER);
- get.addColumn(commitTableFamily, INVALID_TX_QUALIFIER);
-
- Result result = table.get(get);
-
- if (containsInvalidTransaction(result)) {
- CommitTimestamp invalidCT =
- new CommitTimestamp(Location.COMMIT_TABLE, INVALID_TRANSACTION_MARKER, false);
- f.set(Optional.of(invalidCT));
- return f;
- }
-
- if (containsATimestamp(result)) {
- long commitTSValue =
- decodeCommitTimestamp(startTimestamp, result.getValue(commitTableFamily, COMMIT_TABLE_QUALIFIER));
- CommitTimestamp validCT = new CommitTimestamp(Location.COMMIT_TABLE, commitTSValue, true);
- f.set(Optional.of(validCT));
- } else {
- f.set(Optional.<CommitTimestamp>absent());
- }
- } catch (IOException e) {
- LOG.error("Error getting commit timestamp for TX {}", startTimestamp, e);
- f.setException(e);
- }
- return f;
- }
-
- @Override
- public ListenableFuture<Long> readLowWatermark() {
- SettableFuture<Long> f = SettableFuture.create();
- try {
- Get get = new Get(LOW_WATERMARK_ROW);
- get.addColumn(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER);
- Result result = table.get(get);
- if (containsLowWatermark(result)) {
- long lowWatermark = Bytes.toLong(result.getValue(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER));
- f.set(lowWatermark);
- } else {
- f.set(0L);
- }
- } catch (IOException e) {
- LOG.error("Error getting low watermark", e);
- f.setException(e);
- }
- return f;
- }
-
- @Override
- public ListenableFuture<Void> completeTransaction(long startTimestamp) {
- try {
- synchronized (this) {
-
- if (isClosed) {
- SettableFuture<Void> f = SettableFuture.create();
- f.setException(new IOException("Not accepting requests anymore"));
- return f;
- }
-
- DeleteRequest req = new DeleteRequest(
- new Delete(startTimestampToKey(startTimestamp), startTimestamp));
- deleteQueue.put(req);
- return req;
- }
- } catch (IOException ioe) {
- LOG.warn("Error generating timestamp for transaction completion", ioe);
- SettableFuture<Void> f = SettableFuture.create();
- f.setException(ioe);
- return f;
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- SettableFuture<Void> f = SettableFuture.create();
- f.setException(ie);
- return f;
- }
- }
-
- @Override
- public ListenableFuture<Boolean> tryInvalidateTransaction(long startTimestamp) {
- SettableFuture<Boolean> f = SettableFuture.create();
- try {
- byte[] row = startTimestampToKey(startTimestamp);
- Put invalidationPut = new Put(row, startTimestamp);
- invalidationPut.add(commitTableFamily, INVALID_TX_QUALIFIER, null);
-
- // We need to write to the invalid column only if the commit timestamp
- // is empty. This has to be done atomically. Otherwise, if we first
- // check the commit timestamp and right before the invalidation a commit
- // timestamp is added and read by a transaction, then snapshot isolation
- // might not be hold (due to the invalidation)
- // TODO: Decide what we should we do if we can not contact the commit table. loop till succeed???
- boolean result = table.checkAndPut(row, commitTableFamily, COMMIT_TABLE_QUALIFIER, null, invalidationPut);
- f.set(result);
- } catch (IOException ioe) {
- f.setException(ioe);
- }
- return f;
- }
-
- @Override
- @SuppressWarnings("InfiniteLoopStatement")
- public void run() {
- List<DeleteRequest> reqbatch = new ArrayList<>();
- try {
- while (true) {
- DeleteRequest r = deleteQueue.poll();
- if (r == null && reqbatch.size() == 0) {
- r = deleteQueue.take();
- }
-
- if (r != null) {
- reqbatch.add(r);
- }
-
- if (r == null || reqbatch.size() == DELETE_BATCH_SIZE) {
- List<Delete> deletes = new ArrayList<>();
- for (DeleteRequest dr : reqbatch) {
- deletes.add(dr.getDelete());
- }
- try {
- deleteTable.delete(deletes);
- for (DeleteRequest dr : reqbatch) {
- dr.complete();
- }
- } catch (IOException ioe) {
- LOG.warn("Error contacting hbase", ioe);
- for (DeleteRequest dr : reqbatch) {
- dr.error(ioe);
- }
- } finally {
- reqbatch.clear();
- }
- }
- }
- } catch (InterruptedException ie) {
- // Drain the queue and place the exception in the future
- // for those who placed requests
- LOG.warn("Draining delete queue");
- DeleteRequest queuedRequest = deleteQueue.poll();
- while (queuedRequest != null) {
- reqbatch.add(queuedRequest);
- queuedRequest = deleteQueue.poll();
- }
- for (DeleteRequest dr : reqbatch) {
- dr.error(new IOException("HBase CommitTable is going to be closed"));
- }
- reqbatch.clear();
- Thread.currentThread().interrupt();
- } catch (Throwable t) {
- LOG.error("Transaction completion thread threw exception", t);
- }
- }
-
- @Override
- public synchronized void close() throws IOException {
- isClosed = true;
- deleteBatchExecutor.shutdownNow(); // may need to interrupt take
- try {
- if (!deleteBatchExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
- LOG.warn("Delete executor did not shutdown");
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
-
- LOG.warn("Re-Draining delete queue just in case");
- DeleteRequest queuedRequest = deleteQueue.poll();
- while (queuedRequest != null) {
- queuedRequest.error(new IOException("HBase CommitTable is going to be closed"));
- queuedRequest = deleteQueue.poll();
- }
-
- deleteTable.close();
- table.close();
- }
-
- private boolean containsATimestamp(Result result) {
- return (result != null && result.containsColumn(commitTableFamily, COMMIT_TABLE_QUALIFIER));
- }
-
- private boolean containsInvalidTransaction(Result result) {
- return (result != null && result.containsColumn(commitTableFamily, INVALID_TX_QUALIFIER));
- }
-
- private boolean containsLowWatermark(Result result) {
- return (result != null && result.containsColumn(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER));
- }
-
- private class DeleteRequest extends AbstractFuture<Void> {
- final Delete delete;
-
- DeleteRequest(Delete delete) {
- this.delete = delete;
- }
-
- void error(IOException ioe) {
- setException(ioe);
- }
-
- void complete() {
- set(null);
- }
-
- Delete getDelete() {
- return delete;
- }
- }
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public Writer getWriter() throws IOException {
- return new HBaseWriter();
- }
-
- @Override
- public Client getClient() throws IOException {
- return new HBaseClient();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private byte[] startTimestampToKey(long startTimestamp) throws IOException {
- return keygen.startTimestampToKey(startTimestamp);
- }
-
- private static byte[] encodeCommitTimestamp(long startTimestamp, long commitTimestamp) throws IOException {
- assert (startTimestamp < commitTimestamp);
- long diff = commitTimestamp - startTimestamp;
- byte[] bytes = new byte[CodedOutputStream.computeInt64SizeNoTag(diff)];
- CodedOutputStream cos = CodedOutputStream.newInstance(bytes);
- cos.writeInt64NoTag(diff);
- cos.flush();
- return bytes;
-
- }
-
- private static long decodeCommitTimestamp(long startTimestamp, byte[] encodedCommitTimestamp) throws IOException {
- CodedInputStream cis = CodedInputStream.newInstance(encodedCommitTimestamp);
- long diff = cis.readInt64();
- return startTimestamp + diff;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
deleted file mode 100644
index a0687a1..0000000
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableConfig.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import static com.google.common.base.Charsets.UTF_8;
-
-public class HBaseCommitTableConfig {
-
- public static final String COMMIT_TABLE_NAME_KEY = "omid.committable.tablename";
- public static final String COMMIT_TABLE_CF_NAME_KEY = "omid.committable.cfname";
- public static final String COMMIT_TABLE_LWM_CF_NAME_KEY = "omid.committable.lwm.cfname";
-
- public static final String DEFAULT_COMMIT_TABLE_NAME = "OMID_COMMIT_TABLE";
- public static final String DEFAULT_COMMIT_TABLE_CF_NAME = "F";
- public static final String DEFAULT_COMMIT_TABLE_LWM_CF_NAME = "LWF";
-
- static final byte[] COMMIT_TABLE_QUALIFIER = "C".getBytes(UTF_8);
- static final byte[] INVALID_TX_QUALIFIER = "IT".getBytes(UTF_8);
- static final byte[] LOW_WATERMARK_QUALIFIER = "LWC".getBytes(UTF_8);
- static final byte[] LOW_WATERMARK_ROW = "LOW_WATERMARK".getBytes(UTF_8);
-
- // ----------------------------------------------------------------------------------------------------------------
- // Configuration parameters
- // ----------------------------------------------------------------------------------------------------------------
-
- private String tableName = DEFAULT_COMMIT_TABLE_NAME;
- private byte[] commitTableFamily = Bytes.toBytes(DEFAULT_COMMIT_TABLE_CF_NAME);
- private byte[] lowWatermarkFamily = Bytes.toBytes(DEFAULT_COMMIT_TABLE_LWM_CF_NAME);
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters and setters
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getTableName() {
- return tableName;
- }
-
-
- @Inject(optional = true)
- public void setTableName(@Named(COMMIT_TABLE_NAME_KEY) String tableName) {
- this.tableName = tableName;
- }
-
- public byte[] getCommitTableFamily() {
- return commitTableFamily;
- }
-
-
- @Inject(optional = true)
- public void setCommitTableFamily(@Named(COMMIT_TABLE_CF_NAME_KEY) String commitTableFamily) {
- this.commitTableFamily = commitTableFamily.getBytes(UTF_8);
- }
-
- public byte[] getLowWatermarkFamily() {
- return lowWatermarkFamily;
- }
-
- @Inject(optional = true)
- public void setLowWatermarkFamily(@Named(COMMIT_TABLE_LWM_CF_NAME_KEY) String lowWatermarkFamily) {
- this.lowWatermarkFamily = lowWatermarkFamily.getBytes(UTF_8);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java b/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
deleted file mode 100644
index 4c0d617..0000000
--- a/hbase-commit-table/src/main/java/com/yahoo/omid/committable/hbase/HBaseCommitTableStorageModule.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.inject.AbstractModule;
-import com.yahoo.omid.committable.CommitTable;
-import org.apache.hadoop.conf.Configuration;
-
-import javax.inject.Singleton;
-
-public class HBaseCommitTableStorageModule extends AbstractModule {
-
- @Override
- public void configure() {
-
- requireBinding(Configuration.class);
- // HBase commit table creation
- bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
new file mode 100644
index 0000000..3054fe5
--- /dev/null
+++ b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/DefaultHBaseCommitTableStorageModule.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.name.Names;
+import org.apache.omid.HBaseConfigModule;
+
+/**
+ * This class is instantiated by the yaml parser.
+ * Snake_yaml needs a public POJO style class to work properly with all the setters and getters.
+ */
+public class DefaultHBaseCommitTableStorageModule extends AbstractModule {
+
+ private String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
+ private String familyName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME;
+ private String lowWatermarkFamily = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME;
+ private String keytab;
+ private String principal;
+
+ @Override
+ protected void configure() {
+ bindConstant().annotatedWith(Names.named(HBaseCommitTableConfig.COMMIT_TABLE_NAME_KEY)).to(tableName);
+ bindConstant().annotatedWith(Names.named(HBaseCommitTableConfig.COMMIT_TABLE_CF_NAME_KEY)).to(familyName);
+ bindConstant().annotatedWith(Names.named(HBaseCommitTableConfig.COMMIT_TABLE_LWM_CF_NAME_KEY)).to(lowWatermarkFamily);
+ install(new HBaseConfigModule(principal, keytab));
+ install(new HBaseCommitTableStorageModule());
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // WARNING: Do not remove getters/setters, needed by snake_yaml!
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getFamilyName() {
+ return familyName;
+ }
+
+ public void setFamilyName(String familyName) {
+ this.familyName = familyName;
+ }
+
+ public String getLowWatermarkFamily() {
+ return lowWatermarkFamily;
+ }
+
+ public void setLowWatermarkFamily(String lowWatermarkFamily) {
+ this.lowWatermarkFamily = lowWatermarkFamily;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public String getKeytab() {
+ return keytab;
+ }
+
+ public void setKeytab(String keytab) {
+ this.keytab = keytab;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTable.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTable.java b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTable.java
new file mode 100644
index 0000000..059c7d4
--- /dev/null
+++ b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTable.java
@@ -0,0 +1,422 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp.Location;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.omid.committable.hbase.HBaseCommitTableConfig.COMMIT_TABLE_QUALIFIER;
+import static org.apache.omid.committable.hbase.HBaseCommitTableConfig.INVALID_TX_QUALIFIER;
+import static org.apache.omid.committable.hbase.HBaseCommitTableConfig.LOW_WATERMARK_QUALIFIER;
+import static org.apache.omid.committable.hbase.HBaseCommitTableConfig.LOW_WATERMARK_ROW;
+
+public class HBaseCommitTable implements CommitTable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseCommitTable.class);
+
+ private final Configuration hbaseConfig;
+ private final String tableName;
+ private final byte[] commitTableFamily;
+ private final byte[] lowWatermarkFamily;
+ private final KeyGenerator keygen;
+
+ /**
+ * Create a hbase commit table.
+ * Note that we do not take ownership of the passed htable, it is just used to construct the writer and client.
+ */
+ @Inject
+ public HBaseCommitTable(Configuration hbaseConfig, HBaseCommitTableConfig config) {
+ this(hbaseConfig, config, KeyGeneratorImplementations.defaultKeyGenerator());
+ }
+
+ public HBaseCommitTable(Configuration hbaseConfig, HBaseCommitTableConfig config, KeyGenerator keygen) {
+
+ this.hbaseConfig = hbaseConfig;
+ this.tableName = config.getTableName();
+ this.commitTableFamily = config.getCommitTableFamily();
+ this.lowWatermarkFamily = config.getLowWatermarkFamily();
+ this.keygen = keygen;
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Reader and Writer
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private class HBaseWriter implements Writer {
+
+ private static final long INITIAL_LWM_VALUE = -1L;
+ final HTable table;
+ // Our own buffer for operations
+ final List<Put> writeBuffer = new LinkedList<>();
+ volatile long lowWatermarkToStore = INITIAL_LWM_VALUE;
+
+ HBaseWriter() throws IOException {
+ table = new HTable(hbaseConfig, tableName);
+ }
+
+ @Override
+ public void addCommittedTransaction(long startTimestamp, long commitTimestamp) throws IOException {
+ assert (startTimestamp < commitTimestamp);
+ Put put = new Put(startTimestampToKey(startTimestamp), startTimestamp);
+ byte[] value = encodeCommitTimestamp(startTimestamp, commitTimestamp);
+ put.add(commitTableFamily, COMMIT_TABLE_QUALIFIER, value);
+ writeBuffer.add(put);
+ }
+
+ @Override
+ public void updateLowWatermark(long lowWatermark) throws IOException {
+ lowWatermarkToStore = lowWatermark;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ try {
+ addLowWatermarkToStoreToWriteBuffer();
+ table.put(writeBuffer);
+ writeBuffer.clear();
+ } catch (IOException e) {
+ LOG.error("Error flushing data", e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void clearWriteBuffer() {
+ writeBuffer.clear();
+ }
+
+ @Override
+ public void close() throws IOException {
+ clearWriteBuffer();
+ table.close();
+ }
+
+ private void addLowWatermarkToStoreToWriteBuffer() {
+ long lowWatermark = lowWatermarkToStore;
+ if(lowWatermark != INITIAL_LWM_VALUE) {
+ Put put = new Put(LOW_WATERMARK_ROW);
+ put.add(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER, Bytes.toBytes(lowWatermark));
+ writeBuffer.add(put);
+ }
+ }
+
+ }
+
+ class HBaseClient implements Client, Runnable {
+
+ final HTable table;
+ final HTable deleteTable;
+ final ExecutorService deleteBatchExecutor;
+ final BlockingQueue<DeleteRequest> deleteQueue;
+ boolean isClosed = false; // @GuardedBy("this")
+ final static int DELETE_BATCH_SIZE = 1024;
+
+ HBaseClient() throws IOException {
+ table = new HTable(hbaseConfig, tableName);
+ table.setAutoFlush(false, true);
+ deleteTable = new HTable(hbaseConfig, tableName);
+ deleteQueue = new ArrayBlockingQueue<>(DELETE_BATCH_SIZE);
+
+ deleteBatchExecutor = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("omid-completor-%d").build());
+ deleteBatchExecutor.submit(this);
+
+ }
+
+ @Override
+ public ListenableFuture<Optional<CommitTimestamp>> getCommitTimestamp(long startTimestamp) {
+
+ SettableFuture<Optional<CommitTimestamp>> f = SettableFuture.create();
+ try {
+ Get get = new Get(startTimestampToKey(startTimestamp));
+ get.addColumn(commitTableFamily, COMMIT_TABLE_QUALIFIER);
+ get.addColumn(commitTableFamily, INVALID_TX_QUALIFIER);
+
+ Result result = table.get(get);
+
+ if (containsInvalidTransaction(result)) {
+ CommitTimestamp invalidCT =
+ new CommitTimestamp(Location.COMMIT_TABLE, INVALID_TRANSACTION_MARKER, false);
+ f.set(Optional.of(invalidCT));
+ return f;
+ }
+
+ if (containsATimestamp(result)) {
+ long commitTSValue =
+ decodeCommitTimestamp(startTimestamp, result.getValue(commitTableFamily, COMMIT_TABLE_QUALIFIER));
+ CommitTimestamp validCT = new CommitTimestamp(Location.COMMIT_TABLE, commitTSValue, true);
+ f.set(Optional.of(validCT));
+ } else {
+ f.set(Optional.<CommitTimestamp>absent());
+ }
+ } catch (IOException e) {
+ LOG.error("Error getting commit timestamp for TX {}", startTimestamp, e);
+ f.setException(e);
+ }
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Long> readLowWatermark() {
+ SettableFuture<Long> f = SettableFuture.create();
+ try {
+ Get get = new Get(LOW_WATERMARK_ROW);
+ get.addColumn(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER);
+ Result result = table.get(get);
+ if (containsLowWatermark(result)) {
+ long lowWatermark = Bytes.toLong(result.getValue(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER));
+ f.set(lowWatermark);
+ } else {
+ f.set(0L);
+ }
+ } catch (IOException e) {
+ LOG.error("Error getting low watermark", e);
+ f.setException(e);
+ }
+ return f;
+ }
+
+ @Override
+ public ListenableFuture<Void> completeTransaction(long startTimestamp) {
+ try {
+ synchronized (this) {
+
+ if (isClosed) {
+ SettableFuture<Void> f = SettableFuture.create();
+ f.setException(new IOException("Not accepting requests anymore"));
+ return f;
+ }
+
+ DeleteRequest req = new DeleteRequest(
+ new Delete(startTimestampToKey(startTimestamp), startTimestamp));
+ deleteQueue.put(req);
+ return req;
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Error generating timestamp for transaction completion", ioe);
+ SettableFuture<Void> f = SettableFuture.create();
+ f.setException(ioe);
+ return f;
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ SettableFuture<Void> f = SettableFuture.create();
+ f.setException(ie);
+ return f;
+ }
+ }
+
+ @Override
+ public ListenableFuture<Boolean> tryInvalidateTransaction(long startTimestamp) {
+ SettableFuture<Boolean> f = SettableFuture.create();
+ try {
+ byte[] row = startTimestampToKey(startTimestamp);
+ Put invalidationPut = new Put(row, startTimestamp);
+ invalidationPut.add(commitTableFamily, INVALID_TX_QUALIFIER, null);
+
+ // We need to write to the invalid column only if the commit timestamp
+ // is empty. This has to be done atomically. Otherwise, if we first
+ // check the commit timestamp and right before the invalidation a commit
+ // timestamp is added and read by a transaction, then snapshot isolation
+ // might not be hold (due to the invalidation)
+ // TODO: Decide what we should we do if we can not contact the commit table. loop till succeed???
+ boolean result = table.checkAndPut(row, commitTableFamily, COMMIT_TABLE_QUALIFIER, null, invalidationPut);
+ f.set(result);
+ } catch (IOException ioe) {
+ f.setException(ioe);
+ }
+ return f;
+ }
+
+ @Override
+ @SuppressWarnings("InfiniteLoopStatement")
+ public void run() {
+ List<DeleteRequest> reqbatch = new ArrayList<>();
+ try {
+ while (true) {
+ DeleteRequest r = deleteQueue.poll();
+ if (r == null && reqbatch.size() == 0) {
+ r = deleteQueue.take();
+ }
+
+ if (r != null) {
+ reqbatch.add(r);
+ }
+
+ if (r == null || reqbatch.size() == DELETE_BATCH_SIZE) {
+ List<Delete> deletes = new ArrayList<>();
+ for (DeleteRequest dr : reqbatch) {
+ deletes.add(dr.getDelete());
+ }
+ try {
+ deleteTable.delete(deletes);
+ for (DeleteRequest dr : reqbatch) {
+ dr.complete();
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Error contacting hbase", ioe);
+ for (DeleteRequest dr : reqbatch) {
+ dr.error(ioe);
+ }
+ } finally {
+ reqbatch.clear();
+ }
+ }
+ }
+ } catch (InterruptedException ie) {
+ // Drain the queue and place the exception in the future
+ // for those who placed requests
+ LOG.warn("Draining delete queue");
+ DeleteRequest queuedRequest = deleteQueue.poll();
+ while (queuedRequest != null) {
+ reqbatch.add(queuedRequest);
+ queuedRequest = deleteQueue.poll();
+ }
+ for (DeleteRequest dr : reqbatch) {
+ dr.error(new IOException("HBase CommitTable is going to be closed"));
+ }
+ reqbatch.clear();
+ Thread.currentThread().interrupt();
+ } catch (Throwable t) {
+ LOG.error("Transaction completion thread threw exception", t);
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ isClosed = true;
+ deleteBatchExecutor.shutdownNow(); // may need to interrupt take
+ try {
+ if (!deleteBatchExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
+ LOG.warn("Delete executor did not shutdown");
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+
+ LOG.warn("Re-Draining delete queue just in case");
+ DeleteRequest queuedRequest = deleteQueue.poll();
+ while (queuedRequest != null) {
+ queuedRequest.error(new IOException("HBase CommitTable is going to be closed"));
+ queuedRequest = deleteQueue.poll();
+ }
+
+ deleteTable.close();
+ table.close();
+ }
+
+ private boolean containsATimestamp(Result result) {
+ return (result != null && result.containsColumn(commitTableFamily, COMMIT_TABLE_QUALIFIER));
+ }
+
+ private boolean containsInvalidTransaction(Result result) {
+ return (result != null && result.containsColumn(commitTableFamily, INVALID_TX_QUALIFIER));
+ }
+
+ private boolean containsLowWatermark(Result result) {
+ return (result != null && result.containsColumn(lowWatermarkFamily, LOW_WATERMARK_QUALIFIER));
+ }
+
+ private class DeleteRequest extends AbstractFuture<Void> {
+ final Delete delete;
+
+ DeleteRequest(Delete delete) {
+ this.delete = delete;
+ }
+
+ void error(IOException ioe) {
+ setException(ioe);
+ }
+
+ void complete() {
+ set(null);
+ }
+
+ Delete getDelete() {
+ return delete;
+ }
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public Writer getWriter() throws IOException {
+ return new HBaseWriter();
+ }
+
+ @Override
+ public Client getClient() throws IOException {
+ return new HBaseClient();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private byte[] startTimestampToKey(long startTimestamp) throws IOException {
+ return keygen.startTimestampToKey(startTimestamp);
+ }
+
+ private static byte[] encodeCommitTimestamp(long startTimestamp, long commitTimestamp) throws IOException {
+ assert (startTimestamp < commitTimestamp);
+ long diff = commitTimestamp - startTimestamp;
+ byte[] bytes = new byte[CodedOutputStream.computeInt64SizeNoTag(diff)];
+ CodedOutputStream cos = CodedOutputStream.newInstance(bytes);
+ cos.writeInt64NoTag(diff);
+ cos.flush();
+ return bytes;
+
+ }
+
+ private static long decodeCommitTimestamp(long startTimestamp, byte[] encodedCommitTimestamp) throws IOException {
+ CodedInputStream cis = CodedInputStream.newInstance(encodedCommitTimestamp);
+ long diff = cis.readInt64();
+ return startTimestamp + diff;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableConfig.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableConfig.java b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableConfig.java
new file mode 100644
index 0000000..e7887e1
--- /dev/null
+++ b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableConfig.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import static com.google.common.base.Charsets.UTF_8;
+
+public class HBaseCommitTableConfig {
+
+ public static final String COMMIT_TABLE_NAME_KEY = "omid.committable.tablename";
+ public static final String COMMIT_TABLE_CF_NAME_KEY = "omid.committable.cfname";
+ public static final String COMMIT_TABLE_LWM_CF_NAME_KEY = "omid.committable.lwm.cfname";
+
+ public static final String DEFAULT_COMMIT_TABLE_NAME = "OMID_COMMIT_TABLE";
+ public static final String DEFAULT_COMMIT_TABLE_CF_NAME = "F";
+ public static final String DEFAULT_COMMIT_TABLE_LWM_CF_NAME = "LWF";
+
+ static final byte[] COMMIT_TABLE_QUALIFIER = "C".getBytes(UTF_8);
+ static final byte[] INVALID_TX_QUALIFIER = "IT".getBytes(UTF_8);
+ static final byte[] LOW_WATERMARK_QUALIFIER = "LWC".getBytes(UTF_8);
+ static final byte[] LOW_WATERMARK_ROW = "LOW_WATERMARK".getBytes(UTF_8);
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Configuration parameters
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private String tableName = DEFAULT_COMMIT_TABLE_NAME;
+ private byte[] commitTableFamily = Bytes.toBytes(DEFAULT_COMMIT_TABLE_CF_NAME);
+ private byte[] lowWatermarkFamily = Bytes.toBytes(DEFAULT_COMMIT_TABLE_LWM_CF_NAME);
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters and setters
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getTableName() {
+ return tableName;
+ }
+
+
+ @Inject(optional = true)
+ public void setTableName(@Named(COMMIT_TABLE_NAME_KEY) String tableName) {
+ this.tableName = tableName;
+ }
+
+ public byte[] getCommitTableFamily() {
+ return commitTableFamily;
+ }
+
+
+ @Inject(optional = true)
+ public void setCommitTableFamily(@Named(COMMIT_TABLE_CF_NAME_KEY) String commitTableFamily) {
+ this.commitTableFamily = commitTableFamily.getBytes(UTF_8);
+ }
+
+ public byte[] getLowWatermarkFamily() {
+ return lowWatermarkFamily;
+ }
+
+ @Inject(optional = true)
+ public void setLowWatermarkFamily(@Named(COMMIT_TABLE_LWM_CF_NAME_KEY) String lowWatermarkFamily) {
+ this.lowWatermarkFamily = lowWatermarkFamily.getBytes(UTF_8);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableStorageModule.java b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableStorageModule.java
new file mode 100644
index 0000000..a66797d
--- /dev/null
+++ b/hbase-commit-table/src/main/java/org/apache/omid/committable/hbase/HBaseCommitTableStorageModule.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.inject.AbstractModule;
+import org.apache.omid.committable.CommitTable;
+import org.apache.hadoop.conf.Configuration;
+
+import javax.inject.Singleton;
+
+public class HBaseCommitTableStorageModule extends AbstractModule {
+
+ @Override
+ public void configure() {
+
+ requireBinding(Configuration.class);
+ // HBase commit table creation
+ bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java b/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
deleted file mode 100644
index a460c76..0000000
--- a/hbase-commit-table/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTable.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.Client;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.committable.CommitTable.Writer;
-import com.yahoo.omid.committable.hbase.HBaseCommitTable.HBaseClient;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
-import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class TestHBaseCommitTable {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseCommitTable.class);
-
- private static final String TEST_TABLE = "TEST";
-
- private static final TableName TABLE_NAME = TableName.valueOf(TEST_TABLE);
-
- private static HBaseTestingUtility testutil;
- private static MiniHBaseCluster hbasecluster;
- protected static Configuration hbaseConf;
- private static AggregationClient aggregationClient;
- private byte[] commitTableFamily;
- private byte[] lowWatermarkFamily;
-
-
- @BeforeClass
- public void setUpClass() throws Exception {
- // HBase setup
- hbaseConf = HBaseConfiguration.create();
- DefaultHBaseCommitTableStorageModule module = new DefaultHBaseCommitTableStorageModule();
- commitTableFamily = module.getFamilyName().getBytes();
- lowWatermarkFamily = module.getLowWatermarkFamily().getBytes();
- LOG.info("Create hbase");
- testutil = new HBaseTestingUtility(hbaseConf);
- hbasecluster = testutil.startMiniCluster(1);
- aggregationClient = new AggregationClient(hbaseConf);
-
- }
-
- @AfterClass
- public void tearDownClass() throws Exception {
- if (hbasecluster != null) {
- testutil.shutdownMiniCluster();
- }
- }
-
- @BeforeMethod
- public void setUp() throws Exception {
- HBaseAdmin admin = testutil.getHBaseAdmin();
-
- if (!admin.tableExists(TEST_TABLE)) {
- HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
-
- HColumnDescriptor datafam = new HColumnDescriptor(commitTableFamily);
- datafam.setMaxVersions(Integer.MAX_VALUE);
- desc.addFamily(datafam);
-
- HColumnDescriptor lowWatermarkFam = new HColumnDescriptor(lowWatermarkFamily);
- lowWatermarkFam.setMaxVersions(Integer.MAX_VALUE);
- desc.addFamily(lowWatermarkFam);
-
- desc.addCoprocessor("org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
- admin.createTable(desc);
- }
-
- if (admin.isTableDisabled(TEST_TABLE)) {
- admin.enableTable(TEST_TABLE);
- }
- HTableDescriptor[] tables = admin.listTables();
- for (HTableDescriptor t : tables) {
- LOG.info(t.getNameAsString());
- }
- }
-
- @AfterMethod
- public void tearDown() {
- try {
- LOG.info("tearing Down");
- HBaseAdmin admin = testutil.getHBaseAdmin();
- admin.disableTable(TEST_TABLE);
- admin.deleteTable(TEST_TABLE);
-
- } catch (Exception e) {
- LOG.error("Error tearing down", e);
- }
- }
-
- @Test(timeOut = 30_000)
- public void testBasicBehaviour() throws Throwable {
- HBaseCommitTableConfig config = new HBaseCommitTableConfig();
- config.setTableName(TEST_TABLE);
- HBaseCommitTable commitTable = new HBaseCommitTable(hbaseConf, config);
-
- Writer writer = commitTable.getWriter();
- Client client = commitTable.getClient();
-
- // Test that the first time the table is empty
- assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, commitTableFamily));
-
- // Test the successful creation of 1000 txs in the table
- for (int i = 0; i < 1000; i++) {
- writer.addCommittedTransaction(i, i + 1);
- }
- writer.flush();
- assertEquals("Rows should be 1000!", 1000, rowCount(TABLE_NAME, commitTableFamily));
-
- // Test the we get the right commit timestamps for each previously inserted tx
- for (long i = 0; i < 1000; i++) {
- Optional<CommitTimestamp> commitTimestamp = client.getCommitTimestamp(i).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
- long ct = commitTimestamp.get().getValue();
- assertEquals("Commit timestamp should be " + (i + 1), (i + 1), ct);
- }
- assertEquals("Rows should be 1000!", 1000, rowCount(TABLE_NAME, commitTableFamily));
-
- // Test the successful deletion of the 1000 txs
- Future<Void> f;
- for (long i = 0; i < 1000; i++) {
- f = client.completeTransaction(i);
- f.get();
- }
- assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, commitTableFamily));
-
- // Test we don't get a commit timestamp for a non-existent transaction id in the table
- Optional<CommitTimestamp> commitTimestamp = client.getCommitTimestamp(0).get();
- assertFalse("Commit timestamp should not be present", commitTimestamp.isPresent());
-
- // Test that the first time, the low watermark family in table is empty
- assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, lowWatermarkFamily));
-
- // Test the unsuccessful read of the low watermark the first time
- ListenableFuture<Long> lowWatermarkFuture = client.readLowWatermark();
- assertEquals("Low watermark should be 0", Long.valueOf(0), lowWatermarkFuture.get());
-
- // Test the successful update of the low watermark
- for (int lowWatermark = 0; lowWatermark < 1000; lowWatermark++) {
- writer.updateLowWatermark(lowWatermark);
- }
- writer.flush();
- assertEquals("Should there be only one row!", 1, rowCount(TABLE_NAME, lowWatermarkFamily));
-
- // Test the successful read of the low watermark
- lowWatermarkFuture = client.readLowWatermark();
- long lowWatermark = lowWatermarkFuture.get();
- assertEquals("Low watermark should be 999", 999, lowWatermark);
- assertEquals("Should there be only one row!", 1, rowCount(TABLE_NAME, lowWatermarkFamily));
-
- }
-
- @Test(timeOut = 30_000)
- public void testTransactionInvalidation() throws Throwable {
-
- // Prepare test
- final int TX1_ST = 1;
- final int TX1_CT = 2;
- final int TX2_ST = 11;
- final int TX2_CT = 12;
-
- HBaseCommitTableConfig config = new HBaseCommitTableConfig();
- config.setTableName(TEST_TABLE);
- HBaseCommitTable commitTable = new HBaseCommitTable(hbaseConf, config);
-
- // Components under test
- Writer writer = commitTable.getWriter();
- Client client = commitTable.getClient();
-
- // Test that initially the table is empty
- assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, commitTableFamily));
-
- // Test that a transaction can be added properly to the commit table
- writer.addCommittedTransaction(TX1_ST, TX1_CT);
- writer.flush();
- Optional<CommitTimestamp> commitTimestamp = client.getCommitTimestamp(TX1_ST).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
- long ct = commitTimestamp.get().getValue();
- assertEquals("Commit timestamp should be " + TX1_CT, TX1_CT, ct);
-
- // Test that a committed transaction cannot be invalidated and
- // preserves its commit timestamp after that
- boolean wasInvalidated = client.tryInvalidateTransaction(TX1_ST).get();
- assertFalse("Transaction should not be invalidated", wasInvalidated);
-
- commitTimestamp = client.getCommitTimestamp(TX1_ST).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
- ct = commitTimestamp.get().getValue();
- assertEquals("Commit timestamp should be " + TX1_CT, TX1_CT, ct);
-
- // Test that a non-committed transaction can be invalidated...
- wasInvalidated = client.tryInvalidateTransaction(TX2_ST).get();
- assertTrue("Transaction should be invalidated", wasInvalidated);
- commitTimestamp = client.getCommitTimestamp(TX2_ST).get();
- assertTrue(commitTimestamp.isPresent());
- assertFalse(commitTimestamp.get().isValid());
- ct = commitTimestamp.get().getValue();
- assertEquals("Commit timestamp should be " + CommitTable.INVALID_TRANSACTION_MARKER,
- CommitTable.INVALID_TRANSACTION_MARKER, ct);
- // ...and that if it has been already invalidated, it remains
- // invalidated when someone tries to commit it
- writer.addCommittedTransaction(TX2_ST, TX2_CT);
- writer.flush();
- commitTimestamp = client.getCommitTimestamp(TX2_ST).get();
- assertTrue(commitTimestamp.isPresent());
- assertFalse(commitTimestamp.get().isValid());
- ct = commitTimestamp.get().getValue();
- assertEquals("Commit timestamp should be " + CommitTable.INVALID_TRANSACTION_MARKER,
- CommitTable.INVALID_TRANSACTION_MARKER, ct);
-
- // Test that at the end of the test, the commit table contains 2
- // elements, which correspond to the two rows added in the test
- assertEquals("Rows should be 2!", 2, rowCount(TABLE_NAME, commitTableFamily));
-
- }
-
- @Test(timeOut = 30_000)
- public void testClosingClientEmptyQueuesProperly() throws Throwable {
- HBaseCommitTableConfig config = new HBaseCommitTableConfig();
- config.setTableName(TEST_TABLE);
- HBaseCommitTable commitTable = new HBaseCommitTable(hbaseConf, config);
-
- Writer writer = commitTable.getWriter();
- HBaseCommitTable.HBaseClient client = (HBaseClient) commitTable.getClient();
-
- for (int i = 0; i < 1000; i++) {
- writer.addCommittedTransaction(i, i + 1);
- }
- writer.flush();
-
- // Completing first transaction should be fine
- client.completeTransaction(0).get();
- assertEquals("Rows should be 999!", 999, rowCount(TABLE_NAME, commitTableFamily));
-
- // When closing, removing a transaction should throw an EE with an IOException
- client.close();
- try {
- client.completeTransaction(1).get();
- Assert.fail();
- } catch (ExecutionException e) {
- // Expected
- }
- assertEquals("Delete queue size should be 0!", 0, client.deleteQueue.size());
- assertEquals("Rows should be 999!", 999, rowCount(TABLE_NAME, commitTableFamily));
-
- }
-
- private static long rowCount(TableName table, byte[] family) throws Throwable {
- Scan scan = new Scan();
- scan.addFamily(family);
- return aggregationClient.rowCount(table, new LongColumnInterpreter(), scan);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-commit-table/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTable.java
----------------------------------------------------------------------
diff --git a/hbase-commit-table/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTable.java b/hbase-commit-table/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTable.java
new file mode 100644
index 0000000..8629599
--- /dev/null
+++ b/hbase-commit-table/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTable.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.Client;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.committable.CommitTable.Writer;
+import org.apache.omid.committable.hbase.HBaseCommitTable.HBaseClient;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
+import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class TestHBaseCommitTable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestHBaseCommitTable.class);
+
+ private static final String TEST_TABLE = "TEST";
+
+ private static final TableName TABLE_NAME = TableName.valueOf(TEST_TABLE);
+
+ private static HBaseTestingUtility testutil;
+ private static MiniHBaseCluster hbasecluster;
+ protected static Configuration hbaseConf;
+ private static AggregationClient aggregationClient;
+ private byte[] commitTableFamily;
+ private byte[] lowWatermarkFamily;
+
+
+ @BeforeClass
+ public void setUpClass() throws Exception {
+ // HBase setup
+ hbaseConf = HBaseConfiguration.create();
+ DefaultHBaseCommitTableStorageModule module = new DefaultHBaseCommitTableStorageModule();
+ commitTableFamily = module.getFamilyName().getBytes();
+ lowWatermarkFamily = module.getLowWatermarkFamily().getBytes();
+ LOG.info("Create hbase");
+ testutil = new HBaseTestingUtility(hbaseConf);
+ hbasecluster = testutil.startMiniCluster(1);
+ aggregationClient = new AggregationClient(hbaseConf);
+
+ }
+
+ @AfterClass
+ public void tearDownClass() throws Exception {
+ if (hbasecluster != null) {
+ testutil.shutdownMiniCluster();
+ }
+ }
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ HBaseAdmin admin = testutil.getHBaseAdmin();
+
+ if (!admin.tableExists(TEST_TABLE)) {
+ HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+
+ HColumnDescriptor datafam = new HColumnDescriptor(commitTableFamily);
+ datafam.setMaxVersions(Integer.MAX_VALUE);
+ desc.addFamily(datafam);
+
+ HColumnDescriptor lowWatermarkFam = new HColumnDescriptor(lowWatermarkFamily);
+ lowWatermarkFam.setMaxVersions(Integer.MAX_VALUE);
+ desc.addFamily(lowWatermarkFam);
+
+ desc.addCoprocessor("org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
+ admin.createTable(desc);
+ }
+
+ if (admin.isTableDisabled(TEST_TABLE)) {
+ admin.enableTable(TEST_TABLE);
+ }
+ HTableDescriptor[] tables = admin.listTables();
+ for (HTableDescriptor t : tables) {
+ LOG.info(t.getNameAsString());
+ }
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ try {
+ LOG.info("tearing Down");
+ HBaseAdmin admin = testutil.getHBaseAdmin();
+ admin.disableTable(TEST_TABLE);
+ admin.deleteTable(TEST_TABLE);
+
+ } catch (Exception e) {
+ LOG.error("Error tearing down", e);
+ }
+ }
+
+ @Test(timeOut = 30_000)
+ public void testBasicBehaviour() throws Throwable {
+ HBaseCommitTableConfig config = new HBaseCommitTableConfig();
+ config.setTableName(TEST_TABLE);
+ HBaseCommitTable commitTable = new HBaseCommitTable(hbaseConf, config);
+
+ Writer writer = commitTable.getWriter();
+ Client client = commitTable.getClient();
+
+ // Test that the first time the table is empty
+ assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, commitTableFamily));
+
+ // Test the successful creation of 1000 txs in the table
+ for (int i = 0; i < 1000; i++) {
+ writer.addCommittedTransaction(i, i + 1);
+ }
+ writer.flush();
+ assertEquals("Rows should be 1000!", 1000, rowCount(TABLE_NAME, commitTableFamily));
+
+ // Test the we get the right commit timestamps for each previously inserted tx
+ for (long i = 0; i < 1000; i++) {
+ Optional<CommitTimestamp> commitTimestamp = client.getCommitTimestamp(i).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+ long ct = commitTimestamp.get().getValue();
+ assertEquals("Commit timestamp should be " + (i + 1), (i + 1), ct);
+ }
+ assertEquals("Rows should be 1000!", 1000, rowCount(TABLE_NAME, commitTableFamily));
+
+ // Test the successful deletion of the 1000 txs
+ Future<Void> f;
+ for (long i = 0; i < 1000; i++) {
+ f = client.completeTransaction(i);
+ f.get();
+ }
+ assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, commitTableFamily));
+
+ // Test we don't get a commit timestamp for a non-existent transaction id in the table
+ Optional<CommitTimestamp> commitTimestamp = client.getCommitTimestamp(0).get();
+ assertFalse("Commit timestamp should not be present", commitTimestamp.isPresent());
+
+ // Test that the first time, the low watermark family in table is empty
+ assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, lowWatermarkFamily));
+
+ // Test the unsuccessful read of the low watermark the first time
+ ListenableFuture<Long> lowWatermarkFuture = client.readLowWatermark();
+ assertEquals("Low watermark should be 0", Long.valueOf(0), lowWatermarkFuture.get());
+
+ // Test the successful update of the low watermark
+ for (int lowWatermark = 0; lowWatermark < 1000; lowWatermark++) {
+ writer.updateLowWatermark(lowWatermark);
+ }
+ writer.flush();
+ assertEquals("Should there be only one row!", 1, rowCount(TABLE_NAME, lowWatermarkFamily));
+
+ // Test the successful read of the low watermark
+ lowWatermarkFuture = client.readLowWatermark();
+ long lowWatermark = lowWatermarkFuture.get();
+ assertEquals("Low watermark should be 999", 999, lowWatermark);
+ assertEquals("Should there be only one row!", 1, rowCount(TABLE_NAME, lowWatermarkFamily));
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTransactionInvalidation() throws Throwable {
+
+ // Prepare test
+ final int TX1_ST = 1;
+ final int TX1_CT = 2;
+ final int TX2_ST = 11;
+ final int TX2_CT = 12;
+
+ HBaseCommitTableConfig config = new HBaseCommitTableConfig();
+ config.setTableName(TEST_TABLE);
+ HBaseCommitTable commitTable = new HBaseCommitTable(hbaseConf, config);
+
+ // Components under test
+ Writer writer = commitTable.getWriter();
+ Client client = commitTable.getClient();
+
+ // Test that initially the table is empty
+ assertEquals("Rows should be 0!", 0, rowCount(TABLE_NAME, commitTableFamily));
+
+ // Test that a transaction can be added properly to the commit table
+ writer.addCommittedTransaction(TX1_ST, TX1_CT);
+ writer.flush();
+ Optional<CommitTimestamp> commitTimestamp = client.getCommitTimestamp(TX1_ST).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+ long ct = commitTimestamp.get().getValue();
+ assertEquals("Commit timestamp should be " + TX1_CT, TX1_CT, ct);
+
+ // Test that a committed transaction cannot be invalidated and
+ // preserves its commit timestamp after that
+ boolean wasInvalidated = client.tryInvalidateTransaction(TX1_ST).get();
+ assertFalse("Transaction should not be invalidated", wasInvalidated);
+
+ commitTimestamp = client.getCommitTimestamp(TX1_ST).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+ ct = commitTimestamp.get().getValue();
+ assertEquals("Commit timestamp should be " + TX1_CT, TX1_CT, ct);
+
+ // Test that a non-committed transaction can be invalidated...
+ wasInvalidated = client.tryInvalidateTransaction(TX2_ST).get();
+ assertTrue("Transaction should be invalidated", wasInvalidated);
+ commitTimestamp = client.getCommitTimestamp(TX2_ST).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertFalse(commitTimestamp.get().isValid());
+ ct = commitTimestamp.get().getValue();
+ assertEquals("Commit timestamp should be " + CommitTable.INVALID_TRANSACTION_MARKER,
+ CommitTable.INVALID_TRANSACTION_MARKER, ct);
+ // ...and that if it has been already invalidated, it remains
+ // invalidated when someone tries to commit it
+ writer.addCommittedTransaction(TX2_ST, TX2_CT);
+ writer.flush();
+ commitTimestamp = client.getCommitTimestamp(TX2_ST).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertFalse(commitTimestamp.get().isValid());
+ ct = commitTimestamp.get().getValue();
+ assertEquals("Commit timestamp should be " + CommitTable.INVALID_TRANSACTION_MARKER,
+ CommitTable.INVALID_TRANSACTION_MARKER, ct);
+
+ // Test that at the end of the test, the commit table contains 2
+ // elements, which correspond to the two rows added in the test
+ assertEquals("Rows should be 2!", 2, rowCount(TABLE_NAME, commitTableFamily));
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testClosingClientEmptyQueuesProperly() throws Throwable {
+ HBaseCommitTableConfig config = new HBaseCommitTableConfig();
+ config.setTableName(TEST_TABLE);
+ HBaseCommitTable commitTable = new HBaseCommitTable(hbaseConf, config);
+
+ Writer writer = commitTable.getWriter();
+ HBaseCommitTable.HBaseClient client = (HBaseClient) commitTable.getClient();
+
+ for (int i = 0; i < 1000; i++) {
+ writer.addCommittedTransaction(i, i + 1);
+ }
+ writer.flush();
+
+ // Completing first transaction should be fine
+ client.completeTransaction(0).get();
+ assertEquals("Rows should be 999!", 999, rowCount(TABLE_NAME, commitTableFamily));
+
+ // When closing, removing a transaction should throw an EE with an IOException
+ client.close();
+ try {
+ client.completeTransaction(1).get();
+ Assert.fail();
+ } catch (ExecutionException e) {
+ // Expected
+ }
+ assertEquals("Delete queue size should be 0!", 0, client.deleteQueue.size());
+ assertEquals("Rows should be 999!", 999, rowCount(TABLE_NAME, commitTableFamily));
+
+ }
+
+ private static long rowCount(TableName table, byte[] family) throws Throwable {
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return aggregationClient.rowCount(table, new LongColumnInterpreter(), scan);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 7be81f5..d11e642 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -70,7 +70,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -81,7 +81,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java b/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
deleted file mode 100644
index 2b51777..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/HBaseConfigModule.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.tools.hbase.HBaseLogin;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-import java.io.IOException;
-
-public class HBaseConfigModule extends AbstractModule {
-
- private String principal;
- private String keytab;
-
- public HBaseConfigModule(String principal, String keytab) {
- this.principal = principal;
- this.keytab = keytab;
- }
-
- @Override
- protected void configure() {
- }
-
- @Provides
- public Configuration provideHBaseConfig() throws IOException {
- Configuration configuration = HBaseConfiguration.create();
- SecureHBaseConfig secureHBaseConfig = new SecureHBaseConfig();
- secureHBaseConfig.setKeytab(keytab);
- secureHBaseConfig.setPrincipal(principal);
- HBaseLogin.loginIfNeeded(secureHBaseConfig);
- return configuration;
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // IMPORTANT NOTE
- // Overriding equals() and hashCode() is necessary here to allow this module to be installed multiple times.
- // As example, this module can be installed in a configuration using both the HBase Timestamp & Commit Table stores.
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public boolean equals(Object o) {
-
- if (this == o) return true;
-
- if (o == null || getClass() != o.getClass()) return false;
-
- HBaseConfigModule that = (HBaseConfigModule) o;
-
- if (principal != null ? !principal.equals(that.principal) : that.principal != null) return false;
- return keytab != null ? keytab.equals(that.keytab) : that.keytab == null;
-
- }
-
- @Override
- public int hashCode() {
-
- int result = principal != null ? principal.hashCode() : 0;
- result = 31 * result + (keytab != null ? keytab.hashCode() : 0);
- return result;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java b/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
deleted file mode 100644
index 55a12dd..0000000
--- a/hbase-common/src/main/java/com/yahoo/omid/committable/hbase/KeyGenerator.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import java.io.IOException;
-
-/**
- * Implementations of this interface determine how keys are spread in HBase
- */
-public interface KeyGenerator {
- byte[] startTimestampToKey(long startTimestamp) throws IOException;
-
- long keyToStartTimestamp(byte[] key) throws IOException;
-}
[02/52] [abbrv] incubator-omid git commit: Updated source code with
license header
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
index debbb14..13399c5 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
index 88bf7a5..30b51f2 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
index 7dfc2d0..1cd1176 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
index 5baad1f..a10e2d3 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
index 0786638..f238a6b 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
index ae19bc0..d5782fa 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
index 7931b68..09df8da 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java b/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
index 6f9ed76..600dc51 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
index 55351f2..5a4d43c 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
index df5ab91..eab757e 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
index d5ac476..d257deb 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
index 03ea7e0..e20dd06 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
index 886070a..a2b889e 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
index 73972bf..dc28656 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
index c528d05..a3e5653 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
index 982c8c4..735f51c 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
index 56f0c5e..566964a 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
index e3b93c9..2f8717e 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
index 1d5170f..dcfa6ca 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
index 5690f6f..0999065 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
index ebdd1e8..553a15c 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java b/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
index d17b21f..5ea5ffb 100644
--- a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
+++ b/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import com.google.common.collect.Sets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java b/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
index 92e913d..497ec6d 100644
--- a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
+++ b/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.tsoclient;
import org.testng.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java b/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
index 238823c..4fac4c5 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/Cache.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java b/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
index 99f0e9b..fd9b6d7 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/CacheEvaluation.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java b/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
index e132233..74bfa52 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/CommitHashMap.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
index b60c630..7aa1e4f 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/DisruptorModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java b/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
index 7f11b44..a044b04 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/FatalExceptionHandler.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
index 4b40c92..5582861 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/HALeaseManagementModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
index fa6fb28..98d8261 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryCommitTableStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
index 5c5309d..be1c3fc 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/InMemoryTimestampStorageModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java b/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
index fae744e..8736f8d 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManagement.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java b/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
index 2749ad3..e6ff7f1 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java b/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
index 9099552..e140d50 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java b/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
index b41fd08..aec6141 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java b/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
index 64ef45c..c52e96a 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java b/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
index ed648e4..7a32f1e 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java b/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
index cf87b14..47860ae 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java b/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
index d5b6f01..98be866 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
index 77a1c72..4aa7b51 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
index 674416a..2b46b86 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
index 6b31865..f0788ed 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
index 41e0b88..a69700d 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
index 8b5eaaf..2870cb7 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
index ba5f76f..6194b2e 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
index 67d7790..70309b9 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
index a36d8ae..13fbe2d 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java b/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
index afd60f6..7cfd77a 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
index 129474d..ac6abae 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
index cfa1479..3321ef7 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
index 69ffaf7..94e1e34 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
index ece75b1..c20f7a7 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
index 47722ad..62af975 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
index 9487a8c..091dc8b 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java b/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
index 2df6671..4ebf2c6 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
index 6b3d209..6e196d3 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java b/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
index b9db756..345aa83 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
index 48cb747..7677756 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java b/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
index e75d548..3005947 100644
--- a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
+++ b/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/TestUtils.java b/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
index 9068e3b..7c30691 100644
--- a/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
+++ b/tso-server/src/test/java/com/yahoo/omid/TestUtils.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid;
import org.apache.commons.io.IOUtils;
[43/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ZipfianGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ZipfianGenerator.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ZipfianGenerator.java
deleted file mode 100644
index 4f74162..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ZipfianGenerator.java
+++ /dev/null
@@ -1,298 +0,0 @@
-/**
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you
- * may not use this file except in compliance with the License. You
- * may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License. See accompanying
- * LICENSE file.
- */
-
-package com.yahoo.omid.benchmarks.utils;
-
-import java.util.Random;
-
-/**
- * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according
- * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either
- * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of
- * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount).
- *
- * Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most popular, and so on (or min is the most
- * popular, min+1 the next most popular, etc.) If you don't want this clustering, and instead want the popular items scattered throughout the
- * item space, then use ScrambledZipfianGenerator instead.
- *
- * Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a minute
- * for 100 million objects). This is because certain mathematical values need to be computed to properly generate a zipfian skew, and one of those
- * values (zeta) is a sum sequence from 1 to n, where n is the itemcount. Note that if you increase the number of items in the set, we can compute
- * a new zeta incrementally, so it should be fast unless you have added millions of items. However, if you decrease the number of items, we recompute
- * zeta from scratch, so this can take a long time.
- *
- * The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994.
- */
-public class ZipfianGenerator extends IntegerGenerator {
- public static final double ZIPFIAN_CONSTANT = 0.99;
-
- /**
- * Number of items.
- */
- long items;
-
- /**
- * Min item to generate.
- */
- long base;
-
- /**
- * The zipfian constant to use.
- */
- double zipfianconstant;
-
- /**
- * Computed parameters for generating the distribution.
- */
- double alpha, zetan, eta, theta, zeta2theta;
-
- /**
- * The number of items used to compute zetan the last time.
- */
- long countforzeta;
-
- /**
- * Flag to prevent problems. If you increase the number of items the zipfian generator is allowed to choose from, this code will incrementally compute a new zeta
- * value for the larger itemcount. However, if you decrease the number of items, the code computes zeta from scratch; this is expensive for large itemsets.
- * Usually this is not intentional; e.g. one thread thinks the number of items is 1001 and calls "nextLong()" with that item count; then another thread who thinks the
- * number of items is 1000 calls nextLong() with itemcount=1000 triggering the expensive recomputation. (It is expensive for 100 million items, not really for 1000 items.) Why
- * did the second thread think there were only 1000 items? maybe it read the item count before the first thread incremented it. So this flag allows you to say if you really do
- * want that recomputation. If true, then the code will recompute zeta if the itemcount goes down. If false, the code will assume itemcount only goes up, and never recompute.
- */
- boolean allowitemcountdecrease = false;
-
- Random random = new Random();
-
- /******************************* Constructors **************************************/
-
- /**
- * Create a zipfian generator for the specified number of items.
- * @param _items The number of items in the distribution.
- */
- public ZipfianGenerator(long _items) {
- this(0, _items - 1);
- }
-
- /**
- * Create a zipfian generator for items between min and max.
- * @param _min The smallest integer to generate in the sequence.
- * @param _max The largest integer to generate in the sequence.
- */
- public ZipfianGenerator(long _min, long _max) {
- this(_min, _max, ZIPFIAN_CONSTANT);
- }
-
- /**
- * Create a zipfian generator for the specified number of items using the specified zipfian constant.
- *
- * @param _items The number of items in the distribution.
- * @param _zipfianconstant The zipfian constant to use.
- */
- public ZipfianGenerator(long _items, double _zipfianconstant) {
- this(0, _items - 1, _zipfianconstant);
- }
-
- /**
- * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant.
- * @param min The smallest integer to generate in the sequence.
- * @param max The largest integer to generate in the sequence.
- * @param _zipfianconstant The zipfian constant to use.
- */
- public ZipfianGenerator(long min, long max, double _zipfianconstant) {
- this(min, max, _zipfianconstant, zetastatic(max - min + 1, _zipfianconstant));
- }
-
- /**
- * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant, using the precomputed value of zeta.
- *
- * @param min The smallest integer to generate in the sequence.
- * @param max The largest integer to generate in the sequence.
- * @param _zipfianconstant The zipfian constant to use.
- * @param _zetan The precomputed zeta constant.
- */
- public ZipfianGenerator(long min, long max, double _zipfianconstant, double _zetan) {
-
- items = max - min + 1;
- base = min;
- zipfianconstant = _zipfianconstant;
-
- theta = zipfianconstant;
-
- zeta2theta = zeta(2, theta);
-
-
- alpha = 1.0 / (1.0 - theta);
- //zetan=zeta(items,theta);
- zetan = _zetan;
- countforzeta = items;
- eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
-
- //System.out.println("XXXX 3 XXXX");
- nextInt();
- //System.out.println("XXXX 4 XXXX");
- }
-
- /**************************************************************************/
-
- /**
- * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, using the
- * zipfian constant theta. Remember the value of n, so if we change the itemcount, we can recompute zeta.
- *
- * @param n The number of items to compute zeta over.
- * @param theta The zipfian constant.
- */
- double zeta(long n, double theta) {
- countforzeta = n;
- return zetastatic(n, theta);
- }
-
- /**
- * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, using the
- * zipfian constant theta. This is a static version of the function which will not remember n.
- * @param n The number of items to compute zeta over.
- * @param theta The zipfian constant.
- */
- static double zetastatic(long n, double theta) {
- return zetastatic(0, n, theta, 0);
- }
-
- /**
- * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that
- * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of
- * n so that if we change the itemcount, we'll know to recompute zeta.
- *
- * @param st The number of items used to compute the last initialsum
- * @param n The number of items to compute zeta over.
- * @param theta The zipfian constant.
- * @param initialsum The value of zeta we are computing incrementally from.
- */
- double zeta(long st, long n, double theta, double initialsum) {
- countforzeta = n;
- return zetastatic(st, n, theta, initialsum);
- }
-
- /**
- * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that
- * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of
- * n so that if we change the itemcount, we'll know to recompute zeta.
- * @param st The number of items used to compute the last initialsum
- * @param n The number of items to compute zeta over.
- * @param theta The zipfian constant.
- * @param initialsum The value of zeta we are computing incrementally from.
- */
- static double zetastatic(long st, long n, double theta, double initialsum) {
- double sum = initialsum;
- for (long i = st; i < n; i++) {
-
- sum += 1 / (Math.pow(i + 1, theta));
- }
-
- //System.out.println("countforzeta="+countforzeta);
-
- return sum;
- }
-
- /****************************************************************************************/
-
- /**
- * Generate the next item. this distribution will be skewed toward lower integers; e.g. 0 will
- * be the most popular, 1 the next most popular, etc.
- * @param itemcount The number of items in the distribution.
- * @return The next item in the sequence.
- */
- public int nextInt(int itemcount) {
- return (int) nextLong(itemcount);
- }
-
- /**
- * Generate the next item as a long.
- *
- * @param itemcount The number of items in the distribution.
- * @return The next item in the sequence.
- */
- public long nextLong(long itemcount) {
- //from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994
-
- if (itemcount != countforzeta) {
-
- //have to recompute zetan and eta, since they depend on itemcount
- synchronized (this) {
- if (itemcount > countforzeta) {
- //System.err.println("WARNING: Incrementally recomputing Zipfian distribtion. (itemcount="+itemcount+" countforzeta="+countforzeta+")");
-
- //we have added more items. can compute zetan incrementally, which is cheaper
- zetan = zeta(countforzeta, itemcount, theta, zetan);
- eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
- } else if ((itemcount < countforzeta) && (allowitemcountdecrease)) {
- //have to start over with zetan
- //note : for large itemsets, this is very slow. so don't do it!
-
- //TODO: can also have a negative incremental computation, e.g. if you decrease the number of items, then just subtract
- //the zeta sequence terms for the items that went away. This would be faster than recomputing from scratch when the number of items
- //decreases
-
- System.err.println("WARNING: Recomputing Zipfian distribtion. This is slow and should be avoided. (itemcount=" + itemcount + " countforzeta=" + countforzeta + ")");
-
- zetan = zeta(itemcount, theta);
- eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
- }
- }
- }
-
- double u = random.nextDouble();
- double uz = u * zetan;
-
- if (uz < 1.0) {
- return 0;
- }
-
- if (uz < 1.0 + Math.pow(0.5, theta)) {
- return 1;
- }
-
- long ret = base + (long) ((itemcount) * Math.pow(eta * u - eta + 1, alpha));
- setLastInt((int) ret);
- return ret;
- }
-
- /**
- * Return the next value, skewed by the Zipfian distribution. The 0th item will be the most popular, followed by the 1st, followed
- * by the 2nd, etc. (Or, if min != 0, the min-th item is the most popular, the min+1th item the next most popular, etc.) If you want the
- * popular items scattered throughout the item space, use ScrambledZipfianGenerator instead.
- */
- @Override
- public int nextInt() {
- return (int) nextLong(items);
- }
-
- /**
- * Return the next value, skewed by the Zipfian distribution. The 0th item will be the most popular, followed by the 1st, followed
- * by the 2nd, etc. (Or, if min != 0, the min-th item is the most popular, the min+1th item the next most popular, etc.) If you want the
- * popular items scattered throughout the item space, use ScrambledZipfianGenerator instead.
- */
- public long nextLong() {
- return nextLong(items);
- }
-
- /**
- * @todo Implement ZipfianGenerator.mean()
- */
- @Override
- public double mean() {
- throw new UnsupportedOperationException("@todo implement ZipfianGenerator.mean()");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/hbase/HBaseCommitTableTester.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/hbase/HBaseCommitTableTester.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/hbase/HBaseCommitTableTester.java
new file mode 100644
index 0000000..8698c83
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/hbase/HBaseCommitTableTester.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.hbase;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.codahale.metrics.graphite.Graphite;
+import com.codahale.metrics.graphite.GraphiteReporter;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.committable.hbase.KeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.BadRandomKeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.BucketKeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.FullRandomKeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.SeqKeyGenerator;
+import org.apache.omid.tools.hbase.HBaseLogin;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+
+public class HBaseCommitTableTester {
+
+ private static class Config {
+
+ @Parameter(names = "-fullRandomAlgo", description = "Full random algo")
+ boolean fullRandomAlgo = false;
+
+ @Parameter(names = "-badRandomAlgo", description = "The original algo")
+ boolean badRandomAlgo = false;
+
+ @Parameter(names = "-bucketAlgo", description = "Bucketing algorithm")
+ boolean bucketingAlgo = false;
+
+ @Parameter(names = "-seqAlgo", description = "Sequential algorithm")
+ boolean seqAlgo = false;
+
+ @Parameter(names = "-batchSize", description = "batch size")
+ int batchSize = 10000;
+
+ @Parameter(names = "-graphite", description = "graphite server to report to")
+ String graphite = null;
+
+ @ParametersDelegate
+ SecureHBaseConfig loginFlags = new SecureHBaseConfig();
+ }
+
+ @SuppressWarnings("InfiniteLoopStatement")
+ public static void main(String[] args) throws Exception {
+ Config config = new Config();
+ new JCommander(config, args);
+
+ Configuration hbaseConfig = HBaseConfiguration.create();
+
+ final KeyGenerator keygen;
+ if (config.fullRandomAlgo) {
+ keygen = new FullRandomKeyGenerator();
+ } else if (config.badRandomAlgo) {
+ keygen = new BadRandomKeyGenerator();
+ } else if (config.bucketingAlgo) {
+ keygen = new BucketKeyGenerator();
+ } else if (config.seqAlgo) {
+ keygen = new SeqKeyGenerator();
+ } else {
+ throw new IllegalArgumentException("Not supported keygen type");
+ }
+
+ HBaseLogin.loginIfNeeded(config.loginFlags);
+
+ HBaseCommitTableConfig commitTableConfig = new HBaseCommitTableConfig();
+ CommitTable commitTable = new HBaseCommitTable(hbaseConfig, commitTableConfig, keygen);
+
+ CommitTable.Writer writer = commitTable.getWriter();
+
+ MetricRegistry metrics = new MetricRegistry();
+ if (config.graphite != null) {
+ String parts[] = config.graphite.split(":");
+ String host = parts[0];
+ Integer port = Integer.valueOf(parts[1]);
+
+ final Graphite graphite = new Graphite(new InetSocketAddress(host, port));
+ final GraphiteReporter reporter = GraphiteReporter.forRegistry(metrics)
+ .prefixedWith("omid-hbase." + keygen.getClass().getSimpleName())
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .filter(MetricFilter.ALL)
+ .build(graphite);
+ reporter.start(10, TimeUnit.SECONDS);
+ }
+ final ConsoleReporter reporter = ConsoleReporter.forRegistry(metrics)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .build();
+ reporter.start(10, TimeUnit.SECONDS);
+
+ Timer flushTimer = metrics.timer("flush");
+ Meter commitsMeter = metrics.meter("commits");
+
+ int i = 0;
+ long ts = 0;
+ while (true) {
+ writer.addCommittedTransaction(ts++, ts++);
+ if (i++ == config.batchSize) {
+ commitsMeter.mark(i);
+ long start = System.nanoTime();
+ writer.flush();
+ flushTimer.update((System.nanoTime() - start), TimeUnit.NANOSECONDS);
+ i = 0;
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/RawTxRunner.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/RawTxRunner.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/RawTxRunner.java
new file mode 100644
index 0000000..ccb296e
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/RawTxRunner.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.tso;
+
+import com.google.common.util.concurrent.RateLimiter;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import org.apache.omid.benchmarks.utils.IntegerGenerator;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.Counter;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.Timer;
+import org.apache.omid.tso.util.DummyCellIdImpl;
+import org.apache.omid.tso.client.AbortException;
+import org.apache.omid.tso.client.CellId;
+import org.apache.omid.tso.client.OmidClientConfiguration;
+import org.apache.omid.tso.client.TSOClient;
+import org.apache.omid.tso.client.TSOFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static com.codahale.metrics.MetricRegistry.name;
+
+class RawTxRunner implements Runnable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RawTxRunner.class);
+
+ private static volatile int txRunnerCounter = 0;
+ private int txRunnerId = txRunnerCounter++;
+
+ // Config params
+ private final int writesetSize;
+ private final boolean fixedWriteSetSize;
+ private final long commitDelayInMs;
+ private final int percentageOfReadOnlyTxs;
+ private final IntegerGenerator cellIdGenerator;
+ private final Random randomGen;
+
+ // Main elements
+ private final TSOClient tsoClient;
+ private final CommitTable.Client commitTableClient;
+
+ // Asynchronous executor for tx post begin sequence: TimestampListener -> Committer -> CommitListener
+ private final ScheduledExecutorService callbackExec =
+ Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
+ .setNameFormat("tx-runner-" + txRunnerId + "-callback")
+ .setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.error("Thread {} threw exception", t, e);
+ }
+ }).build());
+
+ // Statistics to save
+ private final Timer timestampTimer;
+ private final Timer commitTimer;
+ private final Timer abortTimer;
+ private final Counter errorCounter;
+
+ // Allows to setup a maximum rate for the client in req/sec
+ private final RateLimiter rateLimiter;
+
+ // Is this TxRunner still running?
+ private volatile boolean isRunning = false;
+
+ RawTxRunner(final TSOServerBenchmarkConfig expConfig) throws IOException, InterruptedException {
+
+ // Injector configuration
+ List<Module> guiceModules = new ArrayList<>();
+ guiceModules.add(new Module() {
+ @Override
+ public void configure(Binder binder) {
+ binder.bind(MetricsRegistry.class).toInstance(expConfig.getMetrics());
+ }
+ });
+ guiceModules.add(expConfig.getCommitTableStoreModule());
+ Injector injector = Guice.createInjector(guiceModules);
+
+ // Tx Runner config
+ this.writesetSize = expConfig.getWritesetSize();
+ this.fixedWriteSetSize = expConfig.isFixedWritesetSize();
+ this.commitDelayInMs = expConfig.getCommitDelayInMs();
+ this.percentageOfReadOnlyTxs = expConfig.getPercentageOfReadOnlyTxs();
+ this.cellIdGenerator = expConfig.getCellIdGenerator();
+ this.randomGen = new Random(System.currentTimeMillis() * txRunnerId); // to make it channel dependent
+
+ int txRateInReqPerSec = expConfig.getTxRateInRequestPerSecond();
+ long warmUpPeriodInSecs = expConfig.getWarmUpPeriodInSecs();
+
+ LOG.info("TxRunner-{} [ Tx Rate (Req per Sec) -> {} ]", txRunnerId, txRateInReqPerSec);
+ LOG.info("TxRunner-{} [ Warm Up Period -> {} Secs ]", txRunnerId, warmUpPeriodInSecs);
+ LOG.info("TxRunner-{} [ Cell Id Distribution Generator -> {} ]", txRunnerId, expConfig.getCellIdGenerator().getClass());
+ LOG.info("TxRunner-{} [ Max Tx Size -> {} Fixed: {} ]", txRunnerId, writesetSize, fixedWriteSetSize);
+ LOG.info("TxRunner-{} [ Commit delay -> {} Ms ]", txRunnerId, commitDelayInMs);
+ LOG.info("TxRunner-{} [ % of Read-Only Tx -> {} % ]", txRunnerId, percentageOfReadOnlyTxs);
+
+ // Commit table client initialization
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ this.commitTableClient = commitTable.getClient();
+
+ // Stat initialization
+ MetricsRegistry metrics = injector.getInstance(MetricsRegistry.class);
+ String hostName = InetAddress.getLocalHost().getHostName();
+ this.timestampTimer = metrics.timer(name("tx_runner", Integer.toString(txRunnerId), hostName, "timestamp"));
+ this.commitTimer = metrics.timer(name("tx_runner", Integer.toString(txRunnerId), hostName, "commit"));
+ this.abortTimer = metrics.timer(name("tx_runner", Integer.toString(txRunnerId), hostName, "abort"));
+ this.errorCounter = metrics.counter(name("tx_runner", Integer.toString(txRunnerId), hostName, "errors"));
+ LOG.info("TxRunner-{} [ Metrics provider module -> {} ]", txRunnerId, expConfig.getMetrics().getClass());
+
+ // TSO Client initialization
+ OmidClientConfiguration tsoClientConf = expConfig.getOmidClientConfiguration();
+ this.tsoClient = TSOClient.newInstance(tsoClientConf);
+ LOG.info("TxRunner-{} [ Connection Type {}/Connection String {} ]", txRunnerId,
+ tsoClientConf.getConnectionType(), tsoClientConf.getConnectionString());
+
+ // Limiter for configured request per second
+ this.rateLimiter = RateLimiter.create((double) txRateInReqPerSec, warmUpPeriodInSecs, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void run() {
+
+ isRunning = true;
+
+ while (isRunning) {
+ rateLimiter.acquire();
+ long tsRequestTime = System.nanoTime();
+ final TSOFuture<Long> tsFuture = tsoClient.getNewStartTimestamp();
+ tsFuture.addListener(new TimestampListener(tsFuture, tsRequestTime), callbackExec);
+ }
+
+ shutdown();
+
+ }
+
+ public void stop() {
+ isRunning = false;
+ }
+
+ private void shutdown() {
+
+ try {
+ LOG.info("Finishing TxRunner in 3 secs", txRunnerId);
+ boolean wasSuccess = callbackExec.awaitTermination(3, TimeUnit.SECONDS);
+ if (!wasSuccess) {
+ callbackExec.shutdownNow();
+ }
+ commitTableClient.close();
+ tsoClient.close().get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ // ignore
+ } catch (ExecutionException | IOException e) {
+ // ignore
+ } finally {
+ LOG.info("TxRunner {} finished", txRunnerId);
+ }
+
+ }
+
+ private class TimestampListener implements Runnable {
+
+ final TSOFuture<Long> tsFuture;
+ final long tsRequestTime;
+
+ TimestampListener(TSOFuture<Long> tsFuture, long tsRequestTime) {
+ this.tsFuture = tsFuture;
+ this.tsRequestTime = tsRequestTime;
+ }
+
+ @Override
+ public void run() {
+
+ try {
+ long txId = tsFuture.get();
+ timestampTimer.update(System.nanoTime() - tsRequestTime);
+ if (commitDelayInMs <= 0) {
+ callbackExec.execute(new Committer(txId));
+ } else {
+ callbackExec.schedule(new Committer(txId), commitDelayInMs, TimeUnit.MILLISECONDS);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ errorCounter.inc();
+ } catch (ExecutionException e) {
+ errorCounter.inc();
+ }
+
+ }
+
+ }
+
+ private class Committer implements Runnable {
+
+ final long txId;
+
+ Committer(long txId) {
+ this.txId = txId;
+ }
+
+ @Override
+ public void run() {
+
+ int txWritesetSize = calculateTxWritesetSize();
+
+ if (txWritesetSize == 0) {
+ return; // Read only tx, no need to commit
+ }
+ // Otherwise, we create the writeset...
+ final Set<CellId> cells = new HashSet<>();
+ for (byte i = 0; i < txWritesetSize; i++) {
+ long cellId = cellIdGenerator.nextInt();
+ cells.add(new DummyCellIdImpl(cellId));
+ }
+ // ... and we commit the transaction
+ long startCommitTimeInNs = System.nanoTime();
+ final TSOFuture<Long> commitFuture = tsoClient.commit(txId, cells);
+ commitFuture.addListener(new CommitListener(txId, commitFuture, startCommitTimeInNs), callbackExec);
+
+ }
+
+ private int calculateTxWritesetSize() {
+ int txSize = 0;
+ boolean readOnly = (randomGen.nextFloat() * 100) < percentageOfReadOnlyTxs;
+ if (!readOnly) {
+ if (fixedWriteSetSize) {
+ txSize = writesetSize;
+ } else {
+ txSize = randomGen.nextInt(writesetSize) + 1;
+ }
+ }
+ return txSize;
+ }
+
+ }
+
+ private class CommitListener implements Runnable {
+
+ final long txId;
+ final long commitRequestTime;
+ final TSOFuture<Long> commitFuture;
+
+ CommitListener(long txId, TSOFuture<Long> commitFuture, long commitRequestTime) {
+ this.txId = txId;
+ this.commitFuture = commitFuture;
+ this.commitRequestTime = commitRequestTime;
+ }
+
+ @Override
+ public void run() {
+
+ try {
+ commitFuture.get();
+ commitTableClient.completeTransaction(txId).get();
+ commitTimer.update(System.nanoTime() - commitRequestTime);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ errorCounter.inc();
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof AbortException) {
+ abortTimer.update(System.nanoTime() - commitRequestTime);
+ } else {
+ errorCounter.inc();
+ }
+ }
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmark.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmark.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmark.java
new file mode 100644
index 0000000..15b3a43
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmark.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.tso;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Benchmark using directly TSOClient to connect to the TSO Server
+ */
+public class TSOServerBenchmark implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOServerBenchmark.class);
+
+ private volatile boolean isCleaningDone = false;
+
+ private final TSOServerBenchmarkConfig expConfig;
+
+ // Clients triggering txs (threads) & corresponding executor
+ private final ArrayList<RawTxRunner> txRunners = new ArrayList<>();
+ private final ScheduledExecutorService txRunnerExec;
+
+ private TSOServerBenchmark(TSOServerBenchmarkConfig expConfig) throws IOException {
+
+ this.expConfig = expConfig;
+
+ // Executor for TxRunners (Clients triggering transactions)
+ Thread.UncaughtExceptionHandler uncaughtExceptionHandler = new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ LOG.error("Thread {} threw exception", t, e);
+ }
+ };
+ ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder()
+ .setNameFormat("tx-runner-%d")
+ .setUncaughtExceptionHandler(uncaughtExceptionHandler);
+ this.txRunnerExec = Executors.newScheduledThreadPool(expConfig.getTxRunners(), threadFactoryBuilder.build());
+
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ final TSOServerBenchmarkConfig config = new TSOServerBenchmarkConfig();
+
+ final int nOfTxRunners = config.getTxRunners();
+
+ final long benchmarkRunLengthInMins = config.getBenchmarkRunLengthInMins();
+
+ try (TSOServerBenchmark tsoBenchmark = new TSOServerBenchmark(config)) {
+
+ tsoBenchmark.attachShutDownHook();
+
+ LOG.info("----- Starting TSO Benchmark [ {} TxRunner clients ] -----", nOfTxRunners);
+
+ for (int i = 0; i < nOfTxRunners; ++i) {
+ tsoBenchmark.createTxRunner();
+ }
+
+ LOG.info("Benchmark run lenght {} Mins", benchmarkRunLengthInMins);
+ TimeUnit.MINUTES.sleep(benchmarkRunLengthInMins);
+
+ } finally {
+ LOG.info("----- TSO Benchmark complete - Check metrics from individual clients in log -----");
+ }
+
+ }
+
+ private void attachShutDownHook() {
+ Runtime.getRuntime().addShutdownHook(new Thread("benchmark-cleaner") {
+ @Override
+ public void run() {
+ if (!isCleaningDone) {
+ close();
+ }
+ }
+ });
+ LOG.info("Shutdown Hook Attached");
+ }
+
+ private void createTxRunner() throws IOException, InterruptedException, ExecutionException {
+
+ RawTxRunner txRunner = new RawTxRunner(expConfig);
+ txRunnerExec.submit(txRunner);
+
+ txRunners.add(txRunner);
+
+ }
+
+ @Override
+ public void close() {
+
+ // Stop clients
+ for (RawTxRunner txRunner : txRunners) {
+ txRunner.stop();
+ }
+
+ // Shutdown executor
+ try {
+ LOG.info("Closing TxRunner Executor in 10 secs");
+ boolean wasSuccess = txRunnerExec.awaitTermination(10, TimeUnit.SECONDS);
+ if (!wasSuccess) {
+ txRunnerExec.shutdownNow();
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.info("Interrupted whilst shutting down TxRunner Executor!");
+ } finally {
+ LOG.info("TxRunner Executor stopped");
+ }
+
+ isCleaningDone = true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmarkConfig.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
new file mode 100644
index 0000000..1475d93
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.tso;
+
+import com.google.inject.AbstractModule;
+import org.apache.omid.YAMLUtils;
+import org.apache.omid.benchmarks.utils.IntegerGenerator;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+import org.apache.omid.tso.client.OmidClientConfiguration;
+
+public class TSOServerBenchmarkConfig extends SecureHBaseConfig {
+
+ private static final String DEFAULT_CONFIG_FILE_NAME = "tso-server-benchmark-config.yml";
+ private static final String CONFIG_FILE_NAME = "default-tso-server-benchmark-config.yml";
+
+ private long benchmarkRunLengthInMins;
+
+ private int txRunners;
+ private int txRateInRequestPerSecond;
+ private long warmUpPeriodInSecs;
+ private IntegerGenerator cellIdGenerator;
+ private int writesetSize;
+ private boolean fixedWritesetSize;
+ private int percentageOfReadOnlyTxs;
+ private long commitDelayInMs;
+
+ private OmidClientConfiguration omidClientConfiguration;
+ private AbstractModule commitTableStoreModule;
+
+ private MetricsRegistry metrics;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Instantiation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ TSOServerBenchmarkConfig() {
+ this(CONFIG_FILE_NAME);
+ }
+
+ TSOServerBenchmarkConfig(String configFileName) {
+ new YAMLUtils().loadSettings(DEFAULT_CONFIG_FILE_NAME, configFileName, this);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters and setters for config params
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public long getBenchmarkRunLengthInMins() {
+ return benchmarkRunLengthInMins;
+ }
+
+ public void setBenchmarkRunLengthInMins(long benchmarkRunLengthInMins) {
+ this.benchmarkRunLengthInMins = benchmarkRunLengthInMins;
+ }
+
+ public int getTxRunners() {
+ return txRunners;
+ }
+
+ public void setTxRunners(int txRunners) {
+ this.txRunners = txRunners;
+ }
+
+ public int getTxRateInRequestPerSecond() {
+ return txRateInRequestPerSecond;
+ }
+
+ public void setTxRateInRequestPerSecond(int txRateInRequestPerSecond) {
+ this.txRateInRequestPerSecond = txRateInRequestPerSecond;
+ }
+
+ public long getWarmUpPeriodInSecs() {
+ return warmUpPeriodInSecs;
+ }
+
+ public void setWarmUpPeriodInSecs(long warmUpPeriodInSecs) {
+ this.warmUpPeriodInSecs = warmUpPeriodInSecs;
+ }
+
+ public IntegerGenerator getCellIdGenerator() {
+ return cellIdGenerator;
+ }
+
+ public void setCellIdGenerator(IntegerGenerator cellIdGenerator) {
+ this.cellIdGenerator = cellIdGenerator;
+ }
+
+ public int getWritesetSize() {
+ return writesetSize;
+ }
+
+ public void setWritesetSize(int writesetSize) {
+ this.writesetSize = writesetSize;
+ }
+
+ public boolean isFixedWritesetSize() {
+ return fixedWritesetSize;
+ }
+
+ public void setFixedWritesetSize(boolean fixedWritesetSize) {
+ this.fixedWritesetSize = fixedWritesetSize;
+ }
+
+ public int getPercentageOfReadOnlyTxs() {
+ return percentageOfReadOnlyTxs;
+ }
+
+ public void setPercentageOfReadOnlyTxs(int percentageOfReadOnlyTxs) {
+ this.percentageOfReadOnlyTxs = percentageOfReadOnlyTxs;
+ }
+
+ public long getCommitDelayInMs() {
+ return commitDelayInMs;
+ }
+
+ public void setCommitDelayInMs(long commitDelayInMs) {
+ this.commitDelayInMs = commitDelayInMs;
+ }
+
+ public OmidClientConfiguration getOmidClientConfiguration() {
+ return omidClientConfiguration;
+ }
+
+ public void setOmidClientConfiguration(OmidClientConfiguration omidClientConfiguration) {
+ this.omidClientConfiguration = omidClientConfiguration;
+ }
+
+ public AbstractModule getCommitTableStoreModule() {
+ return commitTableStoreModule;
+ }
+
+ public void setCommitTableStoreModule(AbstractModule commitTableStoreModule) {
+ this.commitTableStoreModule = commitTableStoreModule;
+ }
+
+ public MetricsRegistry getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(MetricsRegistry metrics) {
+ this.metrics = metrics;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/Generator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/Generator.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/Generator.java
new file mode 100644
index 0000000..159d5f9
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/Generator.java
@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.utils;
+
+/**
+ * An expression that generates a sequence of string values, following some distribution (Uniform, Zipfian, Sequential, etc.)
+ */
+public abstract class Generator {
+ /**
+ * Generate the next string in the distribution.
+ */
+ public abstract String nextString();
+
+ /**
+ * Return the previous string generated by the distribution; e.g., returned from the last nextString() call.
+ * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
+ * been called, lastString() should return something reasonable.
+ */
+ public abstract String lastString();
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/IntegerGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/IntegerGenerator.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/IntegerGenerator.java
new file mode 100644
index 0000000..2c5e4a0
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/IntegerGenerator.java
@@ -0,0 +1,65 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.utils;
+
+/**
+ * A generator that is capable of generating ints as well as strings
+ *
+ * @author cooperb
+ *
+ */
+public abstract class IntegerGenerator extends Generator {
+ int lastint;
+
+ /**
+ * Set the last value generated. IntegerGenerator subclasses must use this call
+ * to properly set the last string value, or the lastString() and lastInt() calls won't work.
+ */
+ protected void setLastInt(int last) {
+ lastint = last;
+ }
+
+ /**
+ * Return the next value as an int. When overriding this method, be sure to call setLastString() properly, or the lastString() call won't work.
+ */
+ public abstract int nextInt();
+
+ /**
+ * Generate the next string in the distribution.
+ */
+ public String nextString() {
+ return "" + nextInt();
+ }
+
+ /**
+ * Return the previous string generated by the distribution; e.g., returned from the last nextString() call.
+ * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
+ * been called, lastString() should return something reasonable.
+ */
+ @Override
+ public String lastString() {
+ return "" + lastInt();
+ }
+
+ /**
+ * Return the previous int generated by the distribution. This call is unique to IntegerGenerator subclasses, and assumes
+ * IntegerGenerator subclasses always return ints for nextInt() (e.g. not arbitrary strings).
+ */
+ public int lastInt() {
+ return lastint;
+ }
+
+ /**
+ * Return the expected value (mean) of the values this generator will return.
+ */
+ public abstract double mean();
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ScrambledZipfianGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ScrambledZipfianGenerator.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ScrambledZipfianGenerator.java
new file mode 100644
index 0000000..57cb2c1
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ScrambledZipfianGenerator.java
@@ -0,0 +1,147 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.utils;
+
+
+/**
+ * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according
+ * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either
+ * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of
+ * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount).
+ *
+ * Unlike @ZipfianGenerator, this class scatters the "popular" items across the itemspace. Use this, instead of @ZipfianGenerator, if you
+ * don't want the head of the distribution (the popular items) clustered together.
+ */
+public class ScrambledZipfianGenerator extends IntegerGenerator {
+ public static final double ZETAN = 26.46902820178302;
+ public static final double USED_ZIPFIAN_CONSTANT = 0.99;
+ public static final long ITEM_COUNT = 10000000000L;
+
+ ZipfianGenerator gen;
+ long _min, _max, _itemcount;
+
+ /******************************* Constructors **************************************/
+
+ /**
+ * Create a zipfian generator for the specified number of items.
+ * @param _items The number of items in the distribution.
+ */
+ public ScrambledZipfianGenerator(long _items) {
+ this(0, _items - 1);
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max.
+ * @param _min The smallest integer to generate in the sequence.
+ * @param _max The largest integer to generate in the sequence.
+ */
+ public ScrambledZipfianGenerator(long _min, long _max) {
+ this(_min, _max, ZipfianGenerator.ZIPFIAN_CONSTANT);
+ }
+
+ /**
+ * Create a zipfian generator for the specified number of items using the specified zipfian constant.
+ *
+ * @param _items The number of items in the distribution.
+ * @param _zipfianconstant The zipfian constant to use.
+ */
+ /*
+// not supported, as the value of zeta depends on the zipfian constant, and we have only precomputed zeta for one zipfian constant
+ public ScrambledZipfianGenerator(long _items, double _zipfianconstant)
+ {
+ this(0,_items-1,_zipfianconstant);
+ }
+*/
+
+ /**
+ * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. If you
+ * use a zipfian constant other than 0.99, this will take a long time to complete because we need to recompute zeta.
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ * @param _zipfianconstant The zipfian constant to use.
+ */
+ public ScrambledZipfianGenerator(long min, long max, double _zipfianconstant) {
+ _min = min;
+ _max = max;
+ _itemcount = _max - _min + 1;
+ if (_zipfianconstant == USED_ZIPFIAN_CONSTANT) {
+ gen = new ZipfianGenerator(0, ITEM_COUNT, _zipfianconstant, ZETAN);
+ } else {
+ gen = new ZipfianGenerator(0, ITEM_COUNT, _zipfianconstant);
+ }
+ }
+
+ /**************************************************************************************************/
+
+ /**
+ * Return the next int in the sequence.
+ */
+ @Override
+ public int nextInt() {
+ return (int) nextLong();
+ }
+
+ /**
+ * Return the next long in the sequence.
+ */
+ public long nextLong() {
+ long ret = gen.nextLong();
+ ret = _min + FNVhash64(ret) % _itemcount;
+ setLastInt((int) ret);
+ return ret;
+ }
+
+ public static void main(String[] args) {
+ double newzetan = ZipfianGenerator.zetastatic(ITEM_COUNT, ZipfianGenerator.ZIPFIAN_CONSTANT);
+ System.out.println("zetan: " + newzetan);
+ System.exit(0);
+
+ ScrambledZipfianGenerator gen = new ScrambledZipfianGenerator(10000);
+
+ for (int i = 0; i < 1000000; i++) {
+ System.out.println("" + gen.nextInt());
+ }
+ }
+
+ /**
+ * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range.
+ */
+ @Override
+ public double mean() {
+ return ((double) (((long) _min) + (long) _max)) / 2.0;
+ }
+
+ /**
+ * 64 bit FNV hash. Produces more "random" hashes than (say) String.hashCode().
+ *
+ * @param val The value to hash.
+ * @return The hash value
+ */
+ public static long FNVhash64(long val) {
+ //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash
+ long hashval = FNV_offset_basis_64;
+
+ for (int i = 0; i < 8; i++) {
+ long octet = val & 0x00ff;
+ val = val >> 8;
+
+ hashval = hashval ^ octet;
+ hashval = hashval * FNV_prime_64;
+ //hashval = hashval ^ octet;
+ }
+ return Math.abs(hashval);
+ }
+
+ public static final long FNV_offset_basis_64 = 0xCBF29CE484222325L;
+ public static final long FNV_prime_64 = 1099511628211L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/UniformGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/UniformGenerator.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/UniformGenerator.java
new file mode 100644
index 0000000..7f1437f
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/UniformGenerator.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.utils;
+
+import java.util.Random;
+
+public class UniformGenerator extends IntegerGenerator {
+
+ private final Random ramdom = new Random(System.nanoTime());
+
+ @Override
+ public int nextInt() {
+ return ramdom.nextInt(Integer.MAX_VALUE);
+ }
+
+ @Override
+ public double mean() {
+ return 0;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ZipfianGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ZipfianGenerator.java b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ZipfianGenerator.java
new file mode 100644
index 0000000..44c46ad
--- /dev/null
+++ b/benchmarks/src/main/java/org/apache/omid/benchmarks/utils/ZipfianGenerator.java
@@ -0,0 +1,292 @@
+/**
+ * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.benchmarks.utils;
+
+import java.util.Random;
+
+/**
+ * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according
+ * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either
+ * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of
+ * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount).
+ *
+ * Note that the popular items will be clustered together, e.g. item 0 is the most popular, item 1 the second most popular, and so on (or min is the most
+ * popular, min+1 the next most popular, etc.) If you don't want this clustering, and instead want the popular items scattered throughout the
+ * item space, then use ScrambledZipfianGenerator instead.
+ *
+ * Be aware: initializing this generator may take a long time if there are lots of items to choose from (e.g. over a minute
+ * for 100 million objects). This is because certain mathematical values need to be computed to properly generate a zipfian skew, and one of those
+ * values (zeta) is a sum sequence from 1 to n, where n is the itemcount. Note that if you increase the number of items in the set, we can compute
+ * a new zeta incrementally, so it should be fast unless you have added millions of items. However, if you decrease the number of items, we recompute
+ * zeta from scratch, so this can take a long time.
+ *
+ * The algorithm used here is from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994.
+ */
+public class ZipfianGenerator extends IntegerGenerator {
+ public static final double ZIPFIAN_CONSTANT = 0.99;
+
+ /**
+ * Number of items.
+ */
+ long items;
+
+ /**
+ * Min item to generate.
+ */
+ long base;
+
+ /**
+ * The zipfian constant to use.
+ */
+ double zipfianconstant;
+
+ /**
+ * Computed parameters for generating the distribution.
+ */
+ double alpha, zetan, eta, theta, zeta2theta;
+
+ /**
+ * The number of items used to compute zetan the last time.
+ */
+ long countforzeta;
+
+ /**
+ * Flag to prevent problems. If you increase the number of items the zipfian generator is allowed to choose from, this code will incrementally compute a new zeta
+ * value for the larger itemcount. However, if you decrease the number of items, the code computes zeta from scratch; this is expensive for large itemsets.
+ * Usually this is not intentional; e.g. one thread thinks the number of items is 1001 and calls "nextLong()" with that item count; then another thread who thinks the
+ * number of items is 1000 calls nextLong() with itemcount=1000 triggering the expensive recomputation. (It is expensive for 100 million items, not really for 1000 items.) Why
+ * did the second thread think there were only 1000 items? maybe it read the item count before the first thread incremented it. So this flag allows you to say if you really do
+ * want that recomputation. If true, then the code will recompute zeta if the itemcount goes down. If false, the code will assume itemcount only goes up, and never recompute.
+ */
+ boolean allowitemcountdecrease = false;
+
+ Random random = new Random();
+
+ /******************************* Constructors **************************************/
+
+ /**
+ * Create a zipfian generator for the specified number of items.
+ * @param _items The number of items in the distribution.
+ */
+ public ZipfianGenerator(long _items) {
+ this(0, _items - 1);
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max.
+ * @param _min The smallest integer to generate in the sequence.
+ * @param _max The largest integer to generate in the sequence.
+ */
+ public ZipfianGenerator(long _min, long _max) {
+ this(_min, _max, ZIPFIAN_CONSTANT);
+ }
+
+ /**
+ * Create a zipfian generator for the specified number of items using the specified zipfian constant.
+ *
+ * @param _items The number of items in the distribution.
+ * @param _zipfianconstant The zipfian constant to use.
+ */
+ public ZipfianGenerator(long _items, double _zipfianconstant) {
+ this(0, _items - 1, _zipfianconstant);
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant.
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ * @param _zipfianconstant The zipfian constant to use.
+ */
+ public ZipfianGenerator(long min, long max, double _zipfianconstant) {
+ this(min, max, _zipfianconstant, zetastatic(max - min + 1, _zipfianconstant));
+ }
+
+ /**
+ * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant, using the precomputed value of zeta.
+ *
+ * @param min The smallest integer to generate in the sequence.
+ * @param max The largest integer to generate in the sequence.
+ * @param _zipfianconstant The zipfian constant to use.
+ * @param _zetan The precomputed zeta constant.
+ */
+ public ZipfianGenerator(long min, long max, double _zipfianconstant, double _zetan) {
+
+ items = max - min + 1;
+ base = min;
+ zipfianconstant = _zipfianconstant;
+
+ theta = zipfianconstant;
+
+ zeta2theta = zeta(2, theta);
+
+
+ alpha = 1.0 / (1.0 - theta);
+ //zetan=zeta(items,theta);
+ zetan = _zetan;
+ countforzeta = items;
+ eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
+
+ //System.out.println("XXXX 3 XXXX");
+ nextInt();
+ //System.out.println("XXXX 4 XXXX");
+ }
+
+ /**************************************************************************/
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, using the
+ * zipfian constant theta. Remember the value of n, so if we change the itemcount, we can recompute zeta.
+ *
+ * @param n The number of items to compute zeta over.
+ * @param theta The zipfian constant.
+ */
+ double zeta(long n, double theta) {
+ countforzeta = n;
+ return zetastatic(n, theta);
+ }
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this from scratch for a distribution with n items, using the
+ * zipfian constant theta. This is a static version of the function which will not remember n.
+ * @param n The number of items to compute zeta over.
+ * @param theta The zipfian constant.
+ */
+ static double zetastatic(long n, double theta) {
+ return zetastatic(0, n, theta, 0);
+ }
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that
+ * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of
+ * n so that if we change the itemcount, we'll know to recompute zeta.
+ *
+ * @param st The number of items used to compute the last initialsum
+ * @param n The number of items to compute zeta over.
+ * @param theta The zipfian constant.
+ * @param initialsum The value of zeta we are computing incrementally from.
+ */
+ double zeta(long st, long n, double theta, double initialsum) {
+ countforzeta = n;
+ return zetastatic(st, n, theta, initialsum);
+ }
+
+ /**
+ * Compute the zeta constant needed for the distribution. Do this incrementally for a distribution that
+ * has n items now but used to have st items. Use the zipfian constant theta. Remember the new value of
+ * n so that if we change the itemcount, we'll know to recompute zeta.
+ * @param st The number of items used to compute the last initialsum
+ * @param n The number of items to compute zeta over.
+ * @param theta The zipfian constant.
+ * @param initialsum The value of zeta we are computing incrementally from.
+ */
+ static double zetastatic(long st, long n, double theta, double initialsum) {
+ double sum = initialsum;
+ for (long i = st; i < n; i++) {
+
+ sum += 1 / (Math.pow(i + 1, theta));
+ }
+
+ //System.out.println("countforzeta="+countforzeta);
+
+ return sum;
+ }
+
+ /****************************************************************************************/
+
+ /**
+ * Generate the next item. this distribution will be skewed toward lower integers; e.g. 0 will
+ * be the most popular, 1 the next most popular, etc.
+ * @param itemcount The number of items in the distribution.
+ * @return The next item in the sequence.
+ */
+ public int nextInt(int itemcount) {
+ return (int) nextLong(itemcount);
+ }
+
+ /**
+ * Generate the next item as a long.
+ *
+ * @param itemcount The number of items in the distribution.
+ * @return The next item in the sequence.
+ */
+ public long nextLong(long itemcount) {
+ //from "Quickly Generating Billion-Record Synthetic Databases", Jim Gray et al, SIGMOD 1994
+
+ if (itemcount != countforzeta) {
+
+ //have to recompute zetan and eta, since they depend on itemcount
+ synchronized (this) {
+ if (itemcount > countforzeta) {
+ //System.err.println("WARNING: Incrementally recomputing Zipfian distribtion. (itemcount="+itemcount+" countforzeta="+countforzeta+")");
+
+ //we have added more items. can compute zetan incrementally, which is cheaper
+ zetan = zeta(countforzeta, itemcount, theta, zetan);
+ eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
+ } else if ((itemcount < countforzeta) && (allowitemcountdecrease)) {
+ //have to start over with zetan
+ //note : for large itemsets, this is very slow. so don't do it!
+
+ //TODO: can also have a negative incremental computation, e.g. if you decrease the number of items, then just subtract
+ //the zeta sequence terms for the items that went away. This would be faster than recomputing from scratch when the number of items
+ //decreases
+
+ System.err.println("WARNING: Recomputing Zipfian distribtion. This is slow and should be avoided. (itemcount=" + itemcount + " countforzeta=" + countforzeta + ")");
+
+ zetan = zeta(itemcount, theta);
+ eta = (1 - Math.pow(2.0 / items, 1 - theta)) / (1 - zeta2theta / zetan);
+ }
+ }
+ }
+
+ double u = random.nextDouble();
+ double uz = u * zetan;
+
+ if (uz < 1.0) {
+ return 0;
+ }
+
+ if (uz < 1.0 + Math.pow(0.5, theta)) {
+ return 1;
+ }
+
+ long ret = base + (long) ((itemcount) * Math.pow(eta * u - eta + 1, alpha));
+ setLastInt((int) ret);
+ return ret;
+ }
+
+ /**
+ * Return the next value, skewed by the Zipfian distribution. The 0th item will be the most popular, followed by the 1st, followed
+ * by the 2nd, etc. (Or, if min != 0, the min-th item is the most popular, the min+1th item the next most popular, etc.) If you want the
+ * popular items scattered throughout the item space, use ScrambledZipfianGenerator instead.
+ */
+ @Override
+ public int nextInt() {
+ return (int) nextLong(items);
+ }
+
+ /**
+ * Return the next value, skewed by the Zipfian distribution. The 0th item will be the most popular, followed by the 1st, followed
+ * by the 2nd, etc. (Or, if min != 0, the min-th item is the most popular, the min+1th item the next most popular, etc.) If you want the
+ * popular items scattered throughout the item space, use ScrambledZipfianGenerator instead.
+ */
+ public long nextLong() {
+ return nextLong(items);
+ }
+
+ /**
+ * @todo Implement ZipfianGenerator.mean()
+ */
+ @Override
+ public double mean() {
+ throw new UnsupportedOperationException("@todo implement ZipfianGenerator.mean()");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml b/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
index 5b07bf5..fd2dd58 100644
--- a/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
+++ b/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
@@ -16,10 +16,10 @@ txRateInRequestPerSecond: 100
warmUpPeriodInSecs: 30
# Writeset cell id generation distribution. Options:
-# 1) com.yahoo.omid.benchmarks.utils.UniformGenerator [ ]
-# 2) com.yahoo.omid.benchmarks.utils.ZipfianGenerator [ max_items_as_long_(e.g._Long.MAX_VALUE) ]
-# 3) com.yahoo.omid.benchmarks.utils.ScrambledZipfianGenerator [ max_items_as_long_(e.g._Long.MAX_VALUE) ]
-cellIdGenerator: !!com.yahoo.omid.benchmarks.utils.UniformGenerator [ ]
+# 1) org.apache.omid.benchmarks.utils.UniformGenerator [ ]
+# 2) org.apache.omid.benchmarks.utils.ZipfianGenerator [ max_items_as_long_(e.g._Long.MAX_VALUE) ]
+# 3) org.apache.omid.benchmarks.utils.ScrambledZipfianGenerator [ max_items_as_long_(e.g._Long.MAX_VALUE) ]
+cellIdGenerator: !!org.apache.omid.benchmarks.utils.UniformGenerator [ ]
# Size of tx writesets in terms of number of modified cells
writesetSize: 2
@@ -44,24 +44,24 @@ commitDelayInMs: 0
# ---------------------------------------------------------------------------------------------------------------------
# Omid TSO Client configuration
# ---------------------------------------------------------------------------------------------------------------------
-omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
- connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
+omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration
+ connectionType: !!org.apache.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
connectionString: "localhost:54758"
# ---------------------------------------------------------------------------------------------------------------------
# Commit Table storage configuration
# ---------------------------------------------------------------------------------------------------------------------
-commitTableStoreModule: !!com.yahoo.omid.committable.hbase.DefaultHBaseCommitTableStorageModule
+commitTableStoreModule: !!org.apache.omid.committable.hbase.DefaultHBaseCommitTableStorageModule
tableName: "OMID_COMMIT_TABLE"
# ---------------------------------------------------------------------------------------------------------------------
# Metrics configuration
# ---------------------------------------------------------------------------------------------------------------------
-metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+ !!org.apache.omid.metrics.CodahaleMetricsConfig {
outputFreqInSecs: 10,
reporters: !!set {
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
},
csvDir: "csvMetrics",
}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index bf9f3d9..a34204f 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -3,7 +3,7 @@
<parent>
<artifactId>omid</artifactId>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -17,7 +17,7 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>metrics</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
----------------------------------------------------------------------
diff --git a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java b/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
deleted file mode 100644
index d251b66..0000000
--- a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-import com.google.inject.Inject;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-import java.util.HashSet;
-import java.util.Set;
-
-@Singleton
-public class CodahaleMetricsConfig extends AbstractMetricsConfig {
-
- public enum Reporter {
- CSV, SLF4J, GRAPHITE, CONSOLE
- }
-
- private static final String DEFAULT_PREFIX = "omid";
- private static final String DEFAULT_GRAPHITE_HOST = "localhost:2003";
- private static final String DEFAULT_CSV_DIR = ".";
- private static final String DEFAULT_SLF4J_LOGGER = "metrics";
-
- private static final String METRICS_CODAHALE_PREFIX_KEY = "metrics.codahale.prefix";
- private static final String METRICS_CODAHALE_REPORTERS_KEY = "metrics.codahale.reporters";
- private static final String METRICS_CODAHALE_GRAPHITE_HOST_CONFIG = "metrics.codahale.graphite.host.config";
- private static final String METRICS_CODAHALE_CSV_DIR = "metrics.codahale.cvs.dir";
- private static final String METRICS_CODAHALE_SLF4J_LOGGER = "metrics.codahale.slf4j.logger";
-
- private String prefix = DEFAULT_PREFIX;
- private Set<Reporter> reporters = new HashSet<Reporter>();
- private String graphiteHostConfig = DEFAULT_GRAPHITE_HOST;
- private String csvDir = DEFAULT_CSV_DIR;
- private String slf4jLogger = DEFAULT_SLF4J_LOGGER;
-
- public String getPrefix() {
- return prefix;
- }
-
- @Inject(optional = true)
- public void setPrefix(@Named(METRICS_CODAHALE_PREFIX_KEY) String prefix) {
- this.prefix = prefix;
- }
-
- public Set<Reporter> getReporters() {
- return reporters;
- }
-
- @Inject(optional = true)
- public void setReporters(@Named(METRICS_CODAHALE_REPORTERS_KEY) Set<Reporter> reporters) {
- this.reporters = reporters;
- }
-
- public void addReporter(Reporter reporter) {
- reporters.add(reporter);
- }
-
- public String getGraphiteHostConfig() {
- return graphiteHostConfig;
- }
-
- @Inject(optional = true)
- public void setGraphiteHostConfig(@Named(METRICS_CODAHALE_GRAPHITE_HOST_CONFIG) String graphiteHostConfig) {
- this.graphiteHostConfig = graphiteHostConfig;
- }
-
- public String getCsvDir() {
- return csvDir;
- }
-
- @Inject(optional = true)
- public void setCsvDir(@Named(METRICS_CODAHALE_CSV_DIR) String csvDir) {
- this.csvDir = csvDir;
- }
-
- public String getSlf4jLogger() {
- return slf4jLogger;
- }
-
- @Inject(optional = true)
- public void setSlf4jLogger(@Named(METRICS_CODAHALE_SLF4J_LOGGER) String slf4jLogger) {
- this.slf4jLogger = slf4jLogger;
- }
-
-}
\ No newline at end of file
[21/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
deleted file mode 100644
index ac6abae..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOChannelHandler.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.proto.TSOProto;
-import org.jboss.netty.bootstrap.ServerBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelHandler;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelPipelineFactory;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.Channels;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.group.ChannelGroup;
-import org.jboss.netty.channel.group.DefaultChannelGroup;
-import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
-import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
-import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.channels.ClosedChannelException;
-import java.util.concurrent.Executors;
-
-/**
- * ChannelHandler for the TSO Server.
- *
- * Incoming requests are processed in this class
- */
-public class TSOChannelHandler extends SimpleChannelHandler implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOChannelHandler.class);
-
- private final ChannelFactory factory;
-
- private final ServerBootstrap bootstrap;
-
- @VisibleForTesting
- Channel listeningChannel;
- @VisibleForTesting
- ChannelGroup channelGroup;
-
- private RequestProcessor requestProcessor;
-
- private TSOServerConfig config;
-
- private MetricsRegistry metrics;
-
- @Inject
- public TSOChannelHandler(TSOServerConfig config, RequestProcessor requestProcessor, MetricsRegistry metrics) {
-
- this.config = config;
- this.metrics = metrics;
- this.requestProcessor = requestProcessor;
- // Setup netty listener
- this.factory = new NioServerSocketChannelFactory(
- Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("boss-%d").build()),
- Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("worker-%d").build()),
- (Runtime.getRuntime().availableProcessors() * 2 + 1) * 2);
-
- this.bootstrap = new ServerBootstrap(factory);
- bootstrap.setPipelineFactory(new TSOPipelineFactory(this));
-
- }
-
- /**
- * Allows to create and connect the communication channel closing the previous one if existed
- */
- void reconnect() {
- if (listeningChannel == null && channelGroup == null) {
- LOG.debug("Creating communication channel...");
- } else {
- LOG.debug("Reconnecting communication channel...");
- closeConnection();
- }
- // Create the global ChannelGroup
- channelGroup = new DefaultChannelGroup(TSOChannelHandler.class.getName());
- LOG.debug("\tCreating channel to listening for incoming connections in port {}", config.getPort());
- listeningChannel = bootstrap.bind(new InetSocketAddress(config.getPort()));
- channelGroup.add(listeningChannel);
- LOG.debug("\tListening channel created and connected: {}", listeningChannel);
- }
-
- /**
- * Allows to close the communication channel
- */
- void closeConnection() {
- LOG.debug("Closing communication channel...");
- if (listeningChannel != null) {
- LOG.debug("\tUnbinding listening channel {}", listeningChannel);
- listeningChannel.unbind().awaitUninterruptibly();
- LOG.debug("\tListening channel {} unbound", listeningChannel);
- }
- if (channelGroup != null) {
- LOG.debug("\tClosing channel group {}", channelGroup);
- channelGroup.close().awaitUninterruptibly();
- LOG.debug("\tChannel group {} closed", channelGroup);
- }
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Netty SimpleChannelHandler implementation
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- channelGroup.add(ctx.getChannel());
- LOG.debug("TSO channel connected: {}", ctx.getChannel());
- }
-
- @Override
- public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- channelGroup.remove(ctx.getChannel());
- LOG.debug("TSO channel disconnected: {}", ctx.getChannel());
- }
-
- @Override
- public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- LOG.debug("TSO channel closed: {}", ctx.getChannel());
- }
-
- /**
- * Handle received messages
- */
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
- Object msg = e.getMessage();
- if (msg instanceof TSOProto.Request) {
- TSOProto.Request request = (TSOProto.Request) msg;
- if (request.hasHandshakeRequest()) {
- checkHandshake(ctx, request.getHandshakeRequest());
- return;
- }
- if (!handshakeCompleted(ctx)) {
- LOG.error("Handshake not completed. Closing channel {}", ctx.getChannel());
- ctx.getChannel().close();
- }
-
- if (request.hasTimestampRequest()) {
- requestProcessor.timestampRequest(ctx.getChannel(), new MonitoringContext(metrics));
- } else if (request.hasCommitRequest()) {
- TSOProto.CommitRequest cr = request.getCommitRequest();
- requestProcessor.commitRequest(cr.getStartTimestamp(),
- cr.getCellIdList(),
- cr.getIsRetry(),
- ctx.getChannel(),
- new MonitoringContext(metrics));
- } else {
- LOG.error("Invalid request {}. Closing channel {}", request, ctx.getChannel());
- ctx.getChannel().close();
- }
- } else {
- LOG.error("Unknown message type", msg);
- }
- }
-
- @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
- if (e.getCause() instanceof ClosedChannelException) {
- LOG.warn("ClosedChannelException caught. Cause: ", e.getCause());
- return;
- }
- LOG.warn("Unexpected exception from downstream. Closing channel {}", ctx.getChannel(), e.getCause());
- ctx.getChannel().close();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Closeable implementation
- // ----------------------------------------------------------------------------------------------------------------
- @Override
- public void close() throws IOException {
- closeConnection();
- factory.releaseExternalResources();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods and classes
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * Contains the required context for handshake
- */
- private static class TSOChannelContext {
-
- boolean handshakeComplete;
-
- TSOChannelContext() {
- handshakeComplete = false;
- }
-
- boolean getHandshakeComplete() {
- return handshakeComplete;
- }
-
- void setHandshakeComplete() {
- handshakeComplete = true;
- }
-
- }
-
- private void checkHandshake(final ChannelHandlerContext ctx, TSOProto.HandshakeRequest request) {
-
- TSOProto.HandshakeResponse.Builder response = TSOProto.HandshakeResponse.newBuilder();
- if (request.hasClientCapabilities()) {
-
- response.setClientCompatible(true)
- .setServerCapabilities(TSOProto.Capabilities.newBuilder().build());
- TSOChannelContext tsoCtx = new TSOChannelContext();
- tsoCtx.setHandshakeComplete();
- ctx.setAttachment(tsoCtx);
- } else {
- response.setClientCompatible(false);
- }
- ctx.getChannel().write(TSOProto.Response.newBuilder().setHandshakeResponse(response.build()).build());
-
- }
-
- private boolean handshakeCompleted(ChannelHandlerContext ctx) {
-
- Object o = ctx.getAttachment();
- if (o instanceof TSOChannelContext) {
- TSOChannelContext tsoCtx = (TSOChannelContext) o;
- return tsoCtx.getHandshakeComplete();
- }
- return false;
-
- }
-
- /**
- * Netty pipeline configuration
- */
- static class TSOPipelineFactory implements ChannelPipelineFactory {
-
- private final ChannelHandler handler;
-
- TSOPipelineFactory(ChannelHandler handler) {
- this.handler = handler;
- }
-
- public ChannelPipeline getPipeline() throws Exception {
-
- ChannelPipeline pipeline = Channels.pipeline();
- // Max packet length is 10MB. Transactions with so many cells
- // that the packet is rejected will receive a ServiceUnavailableException.
- // 10MB is enough for 2 million cells in a transaction though.
- pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(10 * 1024 * 1024, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
- pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Request.getDefaultInstance()));
- pipeline.addLast("protobufencoder", new ProtobufEncoder());
- pipeline.addLast("handler", handler);
-
- return pipeline;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
deleted file mode 100644
index 3321ef7..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOModule.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.google.inject.name.Names;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-class TSOModule extends AbstractModule {
- private final TSOServerConfig config;
-
- TSOModule(TSOServerConfig config) {
- this.config = config;
- }
-
- @Override
- protected void configure() {
-
- bind(TSOChannelHandler.class).in(Singleton.class);
- bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
- bind(TimestampOracle.class).to(TimestampOracleImpl.class).in(Singleton.class);
- bind(Panicker.class).to(SystemExitPanicker.class).in(Singleton.class);
-
- // Disruptor setup
- install(new DisruptorModule());
-
- }
-
- @Provides
- TSOServerConfig provideTSOServerConfig() {
- return config;
- }
-
- @Provides
- @Named(TSO_HOST_AND_PORT_KEY)
- String provideTSOHostAndPort() throws SocketException, UnknownHostException {
- return NetworkInterfaceUtils.getTSOHostAndPort(config);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
deleted file mode 100644
index 94e1e34..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServer.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.util.concurrent.AbstractIdleService;
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.Singleton;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.tools.hbase.HBaseLogin;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-@Singleton
-public class TSOServer extends AbstractIdleService {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOServer.class);
-
- public static final String TSO_HOST_AND_PORT_KEY = "tso.hostandport";
-
- @Inject
- private TSOStateManager tsoStateManager;
- @Inject
- private RequestProcessor requestProcessor;
-
- // ----------------------------------------------------------------------------------------------------------------
- // High availability related variables
- // ----------------------------------------------------------------------------------------------------------------
-
- @Inject
- private LeaseManagement leaseManagement;
-
- // ----------------------------------------------------------------------------------------------------------------
-
- static TSOServer getInitializedTsoServer(TSOServerConfig config) throws IOException {
- LOG.info("Configuring TSO Server...");
- Injector injector = Guice.createInjector(buildModuleList(config));
- LOG.info("TSO Server configured. Creating instance...");
- return injector.getInstance(TSOServer.class);
- }
-
- private static List<Module> buildModuleList(final TSOServerConfig config) throws IOException {
-
- List<Module> guiceModules = new ArrayList<>();
- guiceModules.add(config.getTimestampStoreModule());
- guiceModules.add(config.getCommitTableStoreModule());
- guiceModules.add(config.getLeaseModule());
- guiceModules.add(new TSOModule(config));
-
- guiceModules.add(new Module() {
- @Override
- public void configure(Binder binder) {
- LOG.info("\t* Metrics provider module set to {}", config.getMetrics().getClass());
- binder.bind(MetricsRegistry.class).toInstance(config.getMetrics());
- }
- });
- return guiceModules;
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // AbstractIdleService implementation
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- protected void startUp() throws Exception {
- tsoStateManager.register(requestProcessor);
- leaseManagement.startService();
- LOG.info("********** TSO Server running **********");
- }
-
- @Override
- protected void shutDown() throws Exception {
- leaseManagement.stopService();
- tsoStateManager.unregister(requestProcessor);
- LOG.info("********** TSO Server stopped successfully **********");
- }
-
- // ----------------------------------------------------------------------------------------------------------------
-
- private void attachShutDownHook() {
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- stopAndWait();
- }
- });
- LOG.info("Shutdown Hook Attached");
- }
-
- /**
- * This is where all starts on the server side
- */
- public static void main(String[] args) {
-
- TSOServerConfig config = new TSOServerConfig();
-
- try {
- TSOServer tsoServer = getInitializedTsoServer(config);
- tsoServer.attachShutDownHook();
- tsoServer.startAndWait();
- } catch (Exception e) {
- System.out.println(e.getMessage());
- System.exit(-1);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
deleted file mode 100644
index c20f7a7..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOServerConfig.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.inject.Module;
-import com.yahoo.omid.YAMLUtils;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.NetworkInterface;
-import java.net.SocketException;
-import java.util.Enumeration;
-
-/**
- * Reads the configuration parameters of a TSO server instance from CONFIG_FILE_NAME.
- * If file CONFIG_FILE_NAME is missing defaults to DEFAULT_CONFIG_FILE_NAME
- */
-@SuppressWarnings("all")
-public class TSOServerConfig extends SecureHBaseConfig {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOServerConfig.class);
-
- private static final String DEFAULT_CONFIG_FILE_NAME = "default-omid.yml";
- private static final String CONFIG_FILE_NAME = "omid.yml";
-
- private static final String LINUX_TSO_NET_IFACE_PREFIX = "eth";
- private static final String MAC_TSO_NET_IFACE_PREFIX = "en";
-
- // ----------------------------------------------------------------------------------------------------------------
- // Instantiation
- // ----------------------------------------------------------------------------------------------------------------
- public TSOServerConfig() {
- this(CONFIG_FILE_NAME);
- }
-
- @VisibleForTesting
- TSOServerConfig(String configFileName) {
- new YAMLUtils().loadSettings(configFileName, DEFAULT_CONFIG_FILE_NAME, this);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Configuration parameters
- // ----------------------------------------------------------------------------------------------------------------
-
- private Module timestampStoreModule;
-
- private Module commitTableStoreModule;
-
- private Module leaseModule;
-
- private int port;
-
- private MetricsRegistry metrics;
-
- private int maxItems;
-
- private int maxBatchSize;
-
- private int batchPersistTimeoutInMs;
-
- private String networkIfaceName = getDefaultNetworkInterface();
-
- public int getPort() {
- return port;
- }
-
- public void setPort(int port) {
- this.port = port;
- }
-
- public void setMaxBatchSize(int maxBatchSize) {
- this.maxBatchSize = maxBatchSize;
- }
-
- public void setBatchPersistTimeoutInMs(int value) {
- this.batchPersistTimeoutInMs = value;
- }
-
- public String getNetworkIfaceName() {
- return networkIfaceName;
- }
-
- public void setNetworkIfaceName(String networkIfaceName) {
- this.networkIfaceName = networkIfaceName;
- }
-
- public Module getTimestampStoreModule() {
- return timestampStoreModule;
- }
-
- public void setTimestampStoreModule(Module timestampStoreModule) {
- this.timestampStoreModule = timestampStoreModule;
- }
-
- public Module getCommitTableStoreModule() {
- return commitTableStoreModule;
- }
-
- public void setCommitTableStoreModule(Module commitTableStoreModule) {
- this.commitTableStoreModule = commitTableStoreModule;
- }
-
- public Module getLeaseModule() {
- return leaseModule;
- }
-
- public void setLeaseModule(Module leaseModule) {
- this.leaseModule = leaseModule;
- }
-
- public int getMaxItems() {
- return maxItems;
- }
-
- public void setMaxItems(int maxItems) {
- this.maxItems = maxItems;
- }
-
- public int getMaxBatchSize() {
- return maxBatchSize;
- }
-
- public int getBatchPersistTimeoutInMs() {
- return batchPersistTimeoutInMs;
- }
-
- public MetricsRegistry getMetrics() {
- return metrics;
- }
-
- public void setMetrics(MetricsRegistry metrics) {
- this.metrics = metrics;
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private String getDefaultNetworkInterface() {
-
- try {
- Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
- while (networkInterfaces.hasMoreElements()) {
- String name = networkInterfaces.nextElement().getDisplayName();
- LOG.info("Iterating over network interfaces, found '{}'", name);
- if (name.startsWith(MAC_TSO_NET_IFACE_PREFIX) || name.startsWith(LINUX_TSO_NET_IFACE_PREFIX)) {
- return name;
- }
- }
- } catch (SocketException ignored) {
- throw new RuntimeException("Failed to find any network interfaces", ignored);
- }
- throw new IllegalArgumentException(String.format("No network '%s*'/'%s*' interfaces found",
- MAC_TSO_NET_IFACE_PREFIX, LINUX_TSO_NET_IFACE_PREFIX));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
deleted file mode 100644
index 62af975..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManager.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import java.io.IOException;
-
-/**
- * Allows to reset the TSO state and register observers for being notified
- * when changes occur
- */
-public interface TSOStateManager {
-
- /**
- * Represents the state of the TSO
- */
- public static class TSOState {
-
- // TSO state variables
- private final long lowWatermark;
-
- public TSOState(long lowWatermark, long epoch) {
- this.lowWatermark = lowWatermark;
- }
-
- public long getLowWatermark() {
- return lowWatermark;
- }
-
- public long getEpoch() {
- // In this implementation the epoch == low watermark
- return lowWatermark;
- }
-
- @Override
- public String toString() {
- return String.format("LWM %d/Epoch %d", getLowWatermark(), getEpoch());
- }
-
- }
-
- /**
- * Allows implementors to receive the new state when changes occur
- */
- public interface StateObserver {
-
- /**
- * Notifies the observer about the change in state
- * @param state
- * the new TSOState
- */
- public void update(TSOState state) throws IOException;
-
- }
-
- /**
- * Allows to register observers for receiving state changes
- *
- * @param observer
- * the observer to register
- */
- public void register(StateObserver observer);
-
- /**
- * Allows to de-register observers for stopping receiving changes
- *
- * @param observer
- * the observer to unregister
- */
- public void unregister(StateObserver observer);
-
- /**
- * Allows to reset the state
- *
- * @return the new state after reset
- * @throws IOException
- * when problems resetting occur
- */
- public TSOState reset() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
deleted file mode 100644
index 091dc8b..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TSOStateManagerImpl.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Implements the management of the state of the TSO
- */
-public class TSOStateManagerImpl implements TSOStateManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOStateManagerImpl.class);
-
- private List<StateObserver> stateObservers = new ArrayList<>();
-
- private TSOState state;
-
- private TimestampOracle timestampOracle;
-
- @Inject
- public TSOStateManagerImpl(TimestampOracle timestampOracle) {
- this.timestampOracle = timestampOracle;
- }
-
- @Override
- public synchronized void register(StateObserver newObserver) {
- Preconditions.checkNotNull(newObserver, "Trying to register a null observer");
- if (!stateObservers.contains(newObserver)) {
- stateObservers.add(newObserver);
- }
- }
-
- @Override
- public synchronized void unregister(StateObserver observer) {
- stateObservers.remove(observer);
- }
-
- @Override
- public synchronized TSOState reset() throws IOException {
- LOG.info("Reseting the TSO Server state...");
- // The timestamp oracle dictates the new state
- timestampOracle.initialize();
- long lowWatermark = timestampOracle.getLast();
- // In this implementation the epoch == low watermark
- long epoch = lowWatermark;
- state = new TSOState(lowWatermark, epoch);
-
- // Then, notify registered observers about the new state
- for (StateObserver stateObserver : stateObservers) {
- stateObserver.update(state);
- }
- LOG.info("New TSO Server state {}", state);
- return state;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java b/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
deleted file mode 100644
index 4ebf2c6..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracle.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import java.io.IOException;
-
-public interface TimestampOracle {
-
- void initialize() throws IOException;
-
- /**
- * Returns the next timestamp if available. Otherwise spins till the
- * ts-persist thread performs the new timestamp allocation
- */
- long next() throws IOException;
-
- long getLast();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
deleted file mode 100644
index 6e196d3..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TimestampOracleImpl.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.metrics.Gauge;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.io.IOException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-
-/**
- * The Timestamp Oracle that gives monotonically increasing timestamps
- */
-@Singleton
-public class TimestampOracleImpl implements TimestampOracle {
-
- private static final Logger LOG = LoggerFactory.getLogger(TimestampOracleImpl.class);
-
- @VisibleForTesting
- static class InMemoryTimestampStorage implements TimestampStorage {
-
- long maxTimestamp = 0;
-
- @Override
- public void updateMaxTimestamp(long previousMaxTimestamp, long nextMaxTimestamp) {
- maxTimestamp = nextMaxTimestamp;
- LOG.info("Updating max timestamp: (previous:{}, new:{})", previousMaxTimestamp, nextMaxTimestamp);
- }
-
- @Override
- public long getMaxTimestamp() {
- return maxTimestamp;
- }
-
- }
-
- private class AllocateTimestampBatchTask implements Runnable {
- long previousMaxTimestamp;
-
- AllocateTimestampBatchTask(long previousMaxTimestamp) {
- this.previousMaxTimestamp = previousMaxTimestamp;
- }
-
- @Override
- public void run() {
- long newMaxTimestamp = previousMaxTimestamp + TIMESTAMP_BATCH;
- try {
- storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
- maxAllocatedTimestamp = newMaxTimestamp;
- previousMaxTimestamp = newMaxTimestamp;
- } catch (Throwable e) {
- panicker.panic("Can't store the new max timestamp", e);
- }
- }
-
- }
-
- static final long TIMESTAMP_BATCH = 10_000_000; // 10 million
- private static final long TIMESTAMP_REMAINING_THRESHOLD = 1_000_000; // 1 million
-
- private long lastTimestamp;
-
- private long maxTimestamp;
-
- private TimestampStorage storage;
- private Panicker panicker;
-
- private long nextAllocationThreshold;
- private volatile long maxAllocatedTimestamp;
-
- private Executor executor = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("ts-persist-%d").build());
-
- private Runnable allocateTimestampsBatchTask;
-
- @Inject
- public TimestampOracleImpl(MetricsRegistry metrics,
- TimestampStorage tsStorage,
- Panicker panicker) throws IOException {
-
- this.storage = tsStorage;
- this.panicker = panicker;
-
- metrics.gauge(name("tso", "maxTimestamp"), new Gauge<Long>() {
- @Override
- public Long getValue() {
- return maxTimestamp;
- }
- });
-
- }
-
- @Override
- public void initialize() throws IOException {
-
- this.lastTimestamp = this.maxTimestamp = storage.getMaxTimestamp();
-
- this.allocateTimestampsBatchTask = new AllocateTimestampBatchTask(lastTimestamp);
-
- // Trigger first allocation of timestamps
- executor.execute(allocateTimestampsBatchTask);
-
- LOG.info("Initializing timestamp oracle with timestamp {}", this.lastTimestamp);
- }
-
- /**
- * Returns the next timestamp if available. Otherwise spins till the
- * ts-persist thread performs the new timestamp allocation
- */
- @SuppressWarnings("StatementWithEmptyBody")
- @Override
- public long next() throws IOException {
- lastTimestamp++;
-
- if (lastTimestamp == nextAllocationThreshold) {
- executor.execute(allocateTimestampsBatchTask);
- }
-
- if (lastTimestamp >= maxTimestamp) {
- assert (maxTimestamp <= maxAllocatedTimestamp);
- while (maxAllocatedTimestamp == maxTimestamp) {
- // spin
- }
- assert (maxAllocatedTimestamp > maxTimestamp);
- maxTimestamp = maxAllocatedTimestamp;
- nextAllocationThreshold = maxTimestamp - TIMESTAMP_REMAINING_THRESHOLD;
- assert (nextAllocationThreshold > lastTimestamp && nextAllocationThreshold < maxTimestamp);
- assert (lastTimestamp < maxTimestamp);
- }
-
- return lastTimestamp;
- }
-
- @Override
- public long getLast() {
- return lastTimestamp;
- }
-
- @Override
- public String toString() {
- return String.format("TimestampOracle -> LastTimestamp: %d, MaxTimestamp: %d", lastTimestamp, maxTimestamp);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java b/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
deleted file mode 100644
index 345aa83..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/TsoServerDaemon.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.apache.commons.daemon.Daemon;
-import org.apache.commons.daemon.DaemonContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Wraps TSO Server as a Daemon
- */
-public class TsoServerDaemon implements Daemon {
-
- private static final Logger LOG = LoggerFactory.getLogger(TsoServerDaemon.class);
- private TSOServer tsoServer;
-
- @Override
- public void init(DaemonContext daemonContext) throws Exception {
- try {
- TSOServerConfig config = new TSOServerConfig();
- LOG.info("Creating TSOServer instance as a Daemon process...");
- tsoServer = TSOServer.getInitializedTsoServer(config);
- LOG.info("TSOServer instance for Daemon process created");
- } catch (Exception e) {
- LOG.error("Error creating TSOServer instance", e);
- throw e;
- }
- }
-
- @Override
- public void start() throws Exception {
- tsoServer.startAndWait();
- }
-
- @Override
- public void stop() throws Exception {
- tsoServer.stopAndWait();
- }
-
- @Override
- public void destroy() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java b/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
deleted file mode 100644
index 7677756..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManagementModule.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.google.inject.name.Names;
-import com.yahoo.omid.HBaseConfigModule;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig;
-import com.yahoo.omid.tso.LeaseManagement.LeaseManagementException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Singleton;
-
-import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
-
-public class VoidLeaseManagementModule extends AbstractModule {
- private static final Logger LOG = LoggerFactory.getLogger(VoidLeaseManagementModule.class);
- private String keytab;
- private String principal;
- private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
-
- @Override
- protected void configure() {
- LOG.info(this.toString());
- bindConstant().annotatedWith(Names.named(HBaseTimestampStorageConfig.TIMESTAMP_STORAGE_TABLE_NAME_KEY)).to(tableName);
- install(new HBaseConfigModule(principal, keytab));
- }
-
- @Provides
- @Singleton
- LeaseManagement provideLeaseManager(TSOChannelHandler tsoChannelHandler, TSOStateManager stateManager)
- throws LeaseManagementException {
-
- return new VoidLeaseManager(tsoChannelHandler, stateManager);
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // WARNING: Do not remove getters/setters, needed by snake_yaml!
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public String getPrincipal() {
- return principal;
- }
-
- public void setPrincipal(String principal) {
- this.principal = principal;
- }
-
- public String getKeytab() {
- return keytab;
- }
-
- public void setKeytab(String keytab) {
- this.keytab = keytab;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("VoidLeaseManagementModule{");
- sb.append("keytab='").append(keytab).append('\'');
- sb.append(", principal='").append(principal).append('\'');
- sb.append(", tableName='").append(tableName).append('\'');
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java b/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
deleted file mode 100644
index 3005947..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/VoidLeaseManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import java.io.IOException;
-
-public class VoidLeaseManager implements LeaseManagement {
-
- private final TSOChannelHandler tsoChannelHandler;
- private TSOStateManager stateManager;
-
- public VoidLeaseManager(TSOChannelHandler tsoChannelHandler, TSOStateManager stateManager) {
- this.tsoChannelHandler = tsoChannelHandler;
- this.stateManager = stateManager;
- }
-
- @Override
- public void startService() throws LeaseManagementException {
- try {
- stateManager.reset();
- tsoChannelHandler.reconnect();
- } catch (IOException e) {
- throw new LeaseManagementException("Error initializing Lease Manager", e);
- }
- }
-
- @Override
- public void stopService() throws LeaseManagementException {
- tsoChannelHandler.closeConnection();
- }
-
- @Override
- public boolean stillInLeasePeriod() {
- // We should always return true
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/Cache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/Cache.java b/tso-server/src/main/java/org/apache/omid/tso/Cache.java
new file mode 100644
index 0000000..2304fbc
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/Cache.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+/**
+ * This interface defines a container for long-long mappings, for instance for holding row -> last commit information
+ *
+ */
+public interface Cache {
+
+ public void reset();
+
+ public abstract long set(long key, long value);
+
+ public abstract long get(long key);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/CacheEvaluation.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/CacheEvaluation.java b/tso-server/src/main/java/org/apache/omid/tso/CacheEvaluation.java
new file mode 100644
index 0000000..77e9259
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/CacheEvaluation.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import java.io.FileNotFoundException;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.util.Random;
+
+public class CacheEvaluation {
+
+ private final static int ENTRIES = 1000000;
+ private final static int WARMUP_ROUNDS = 2;
+ private final static int ROUNDS = 4;
+ private final static double HOT_PERC = 1;
+
+ public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
+ int[] asoc = new int[]{8, 16, 32};
+ for (int anAsoc : asoc) {
+ PrintWriter writer = new PrintWriter(anAsoc + ".out", "UTF-8");
+ new CacheEvaluation().testEntriesAge(new LongCache(ENTRIES, anAsoc), writer);
+ writer.close();
+ }
+ PrintWriter writer = new PrintWriter("guava.out", "UTF-8");
+ writer.close();
+ }
+
+ private void testEntriesAge(Cache cache, PrintWriter writer) {
+ Random random = new Random();
+
+ long seed = random.nextLong();
+
+ writer.println("# Random seed: " + seed);
+ random.setSeed(seed);
+ int removals = 0;
+ double tempStdDev = 0;
+ double tempAvg = 0;
+
+ int i = 0;
+ int largestDeletedTimestamp = 0;
+ long hotItem = random.nextLong();
+
+ Runtime.getRuntime().gc();
+
+ for (; i < ENTRIES * WARMUP_ROUNDS; ++i) {
+ long toInsert = random.nextInt(100) < HOT_PERC ? hotItem : random.nextLong();
+ long removed = cache.set(toInsert, i);
+ if (removed > largestDeletedTimestamp) {
+ largestDeletedTimestamp = (int) removed;
+ }
+ if (removed > largestDeletedTimestamp) {
+ largestDeletedTimestamp = (int) removed;
+ }
+ if (i % ENTRIES == 0) {
+ int round = i / ENTRIES + 1;
+ System.err.format("Warmup [%d/%d]\n", round, WARMUP_ROUNDS);
+ }
+ }
+
+ long time = System.nanoTime();
+ for (; i < ENTRIES * (WARMUP_ROUNDS + ROUNDS); ++i) {
+ long toInsert = random.nextInt(100) < HOT_PERC ? hotItem : random.nextLong();
+ long removed = cache.set(toInsert, i);
+ if (removed > largestDeletedTimestamp) {
+ largestDeletedTimestamp = (int) removed;
+ }
+ int gap = i - largestDeletedTimestamp;
+ removals++;
+ double oldAvg = tempAvg;
+ tempAvg += (gap - tempAvg) / removals;
+ tempStdDev += (gap - oldAvg) * (gap - tempAvg);
+ if (i % ENTRIES == 0) {
+ int round = i / ENTRIES - WARMUP_ROUNDS + 1;
+ System.err.format("Progress [%d/%d]\n", round, ROUNDS);
+ }
+ }
+ long elapsed = System.nanoTime() - time;
+ double elapsedSeconds = (elapsed / (double) 1000000000);
+ long totalOps = ENTRIES * ROUNDS;
+ writer.println("# Free mem before GC (MB) :" + (Runtime.getRuntime().freeMemory() / (double) (1024 * 1024)));
+ Runtime.getRuntime().gc();
+ writer.println("# Free mem (MB) :" + (Runtime.getRuntime().freeMemory() / (double) (1024 * 1024)));
+ writer.println("# Elapsed (s): " + elapsedSeconds);
+ writer.println("# Elapsed per 100 ops (ms): " + (elapsed / (double) totalOps / 100 / (double) 1000000));
+ writer.println("# Ops per s : " + (totalOps / elapsedSeconds));
+ writer.println("# Avg gap: " + (tempAvg));
+ writer.println("# Std dev gap: " + Math.sqrt((tempStdDev / ENTRIES)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/CommitHashMap.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/CommitHashMap.java b/tso-server/src/main/java/org/apache/omid/tso/CommitHashMap.java
new file mode 100644
index 0000000..c281536
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/CommitHashMap.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Stores the mapping between a particular cell id and the commit timestamp
+ * of the last transaction that changed it.
+ *
+ * The mapping is implemented as a long -> long mapping, using a single long [].
+ * For a map of size N we create an array of size 2*N and store the keys
+ * on even indexes and values on odd indexes. The rationale is that we want
+ * queries to be fast and touch as least memory regions as possible.
+ *
+ * Each time an entry is removed, the caller updates the largestDeletedTimestamp
+ * if the entry's commit timestamp is greater than this value.
+ *
+ * TODO: improve garbage collection, right now an entry is picked at random
+ * (by hash) which could cause the eviction of a very recent timestamp
+ */
+
+class CommitHashMap {
+
+ private static final Logger LOG = LoggerFactory.getLogger(CommitHashMap.class);
+
+ private final Cache cellIdToCommitMap;
+
+ /**
+ * Constructs a new, empty hashtable with a default size of 1000
+ */
+ public CommitHashMap() {
+ this(1000);
+ }
+
+ /**
+ * Constructs a new, empty hashtable with the specified size
+ *
+ * @param size
+ * the initial size of the hashtable.
+ * @throws IllegalArgumentException
+ * if the size is less than zero.
+ */
+ public CommitHashMap(int size) {
+ if (size < 0) {
+ throw new IllegalArgumentException("Illegal size: " + size);
+ }
+
+ this.cellIdToCommitMap = new LongCache(size, 32);
+ LOG.info("CellId -> CommitTS map created with [{}] buckets (32 elems/bucket)", size);
+ }
+
+ public void reset() {
+ cellIdToCommitMap.reset();
+ LOG.info("CellId -> CommitTS map reset");
+ }
+
+ public long getLatestWriteForCell(long hash) {
+ return cellIdToCommitMap.get(hash);
+ }
+
+ public long putLatestWriteForCell(long hash, long commitTimestamp) {
+ return cellIdToCommitMap.set(hash, commitTimestamp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/DisruptorModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/DisruptorModule.java b/tso-server/src/main/java/org/apache/omid/tso/DisruptorModule.java
new file mode 100644
index 0000000..9f90d10
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/DisruptorModule.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+
+import javax.inject.Singleton;
+
+public class DisruptorModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+
+ bind(RequestProcessor.class).to(RequestProcessorImpl.class).in(Singleton.class);
+ bind(PersistenceProcessor.class).to(PersistenceProcessorImpl.class).in(Singleton.class);
+ bind(ReplyProcessor.class).to(ReplyProcessorImpl.class).in(Singleton.class);
+ bind(RetryProcessor.class).to(RetryProcessorImpl.class).in(Singleton.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/FatalExceptionHandler.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/FatalExceptionHandler.java b/tso-server/src/main/java/org/apache/omid/tso/FatalExceptionHandler.java
new file mode 100644
index 0000000..016c1b7
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/FatalExceptionHandler.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.lmax.disruptor.ExceptionHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FatalExceptionHandler implements ExceptionHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(FatalExceptionHandler.class);
+
+ Panicker panicker;
+
+ FatalExceptionHandler(Panicker panicker) {
+ this.panicker = panicker;
+ }
+
+ @Override
+ public void handleEventException(Throwable ex,
+ long sequence,
+ Object event) {
+ LOG.error("Uncaught exception throws for sequence {}, event {}",
+ new Object[]{sequence, event, ex});
+ panicker.panic("Uncaught exception in disruptor thread", ex);
+ }
+
+ @Override
+ public void handleOnShutdownException(Throwable ex) {
+ LOG.warn("Uncaught exception shutting down", ex);
+ }
+
+ @Override
+ public void handleOnStartException(Throwable ex) {
+ panicker.panic("Uncaught exception starting up", ex);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/HALeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/HALeaseManagementModule.java b/tso-server/src/main/java/org/apache/omid/tso/HALeaseManagementModule.java
new file mode 100644
index 0000000..854e3df
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/HALeaseManagementModule.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.timestamp.storage.ZKModule;
+import org.apache.omid.tso.LeaseManagement.LeaseManagementException;
+import org.apache.curator.framework.CuratorFramework;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+import static org.apache.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
+
+public class HALeaseManagementModule extends AbstractModule {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HALeaseManagementModule.class);
+ private final long leasePeriodInMs;
+ private final String tsoLeasePath;
+ private final String currentTsoPath;
+ private final String zkCluster;
+ private final String zkNamespace;
+
+ public HALeaseManagementModule(long leasePeriodInMs, String tsoLeasePath, String currentTsoPath,
+ String zkCluster, String zkNamespace) {
+
+ this.leasePeriodInMs = leasePeriodInMs;
+ this.tsoLeasePath = tsoLeasePath;
+ this.currentTsoPath = currentTsoPath;
+ this.zkCluster = zkCluster;
+ this.zkNamespace = zkNamespace;
+
+ }
+
+ @Override
+ protected void configure() {
+
+ install(new ZKModule(zkCluster, zkNamespace));
+
+ }
+
+ @Provides
+ @Singleton
+ LeaseManagement provideLeaseManager(@Named(TSO_HOST_AND_PORT_KEY) String tsoHostAndPort,
+ TSOChannelHandler tsoChannelHandler,
+ TSOStateManager stateManager,
+ CuratorFramework zkClient,
+ Panicker panicker) throws LeaseManagementException {
+
+ LOG.info("Connection to HA cluster [{}]", zkClient.getState());
+
+ return new LeaseManager(tsoHostAndPort,
+ tsoChannelHandler,
+ stateManager,
+ leasePeriodInMs,
+ tsoLeasePath,
+ currentTsoPath,
+ zkClient,
+ panicker);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/InMemoryCommitTableStorageModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/InMemoryCommitTableStorageModule.java b/tso-server/src/main/java/org/apache/omid/tso/InMemoryCommitTableStorageModule.java
new file mode 100644
index 0000000..ac1c8b8
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/InMemoryCommitTableStorageModule.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.NullCommitTable;
+
+import javax.inject.Singleton;
+
+public class InMemoryCommitTableStorageModule extends AbstractModule {
+
+ @Override
+ public void configure() {
+
+ bind(CommitTable.class).to(NullCommitTable.class).in(Singleton.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/InMemoryTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/InMemoryTimestampStorageModule.java b/tso-server/src/main/java/org/apache/omid/tso/InMemoryTimestampStorageModule.java
new file mode 100644
index 0000000..c234248
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/InMemoryTimestampStorageModule.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.inject.AbstractModule;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.apache.omid.tso.TimestampOracleImpl.InMemoryTimestampStorage;
+
+import javax.inject.Singleton;
+
+public class InMemoryTimestampStorageModule extends AbstractModule {
+
+ @Override
+ public void configure() {
+
+ bind(TimestampStorage.class).to(InMemoryTimestampStorage.class).in(Singleton.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/LeaseManagement.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/LeaseManagement.java b/tso-server/src/main/java/org/apache/omid/tso/LeaseManagement.java
new file mode 100644
index 0000000..dc10b77
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/LeaseManagement.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+public interface LeaseManagement {
+
+ class LeaseManagementException extends Exception {
+
+ private static final long serialVersionUID = -2061376444591776881L;
+
+ LeaseManagementException(String msg) {
+ super(msg);
+ }
+
+
+ LeaseManagementException(String msg, Exception e) {
+ super(msg, e);
+ }
+
+ }
+
+ /**
+ * Allows to start the service implementing the lease management
+ */
+ void startService() throws LeaseManagementException;
+
+ /**
+ * Allows to stop the service implementing the lease management
+ */
+ void stopService() throws LeaseManagementException;
+
+ /**
+ * Check if the instance is still is under the lease period
+ */
+ boolean stillInLeasePeriod();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/org/apache/omid/tso/LeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/org/apache/omid/tso/LeaseManager.java b/tso-server/src/main/java/org/apache/omid/tso/LeaseManager.java
new file mode 100644
index 0000000..738f593
--- /dev/null
+++ b/tso-server/src/main/java/org/apache/omid/tso/LeaseManager.java
@@ -0,0 +1,310 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.tso.TSOStateManager.TSOState;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.utils.EnsurePath;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Encompasses all the required elements to control the leases required for
+ * identifying the master instance when running multiple TSO instances for HA
+ * It delegates the initialization of the TSO state and the publication of
+ * the instance information when getting the lease to an asynchronous task to
+ * continue managing the leases without interruptions.
+ */
+class LeaseManager extends AbstractScheduledService implements LeaseManagement {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LeaseManager.class);
+
+ private final CuratorFramework zkClient;
+
+ private final Panicker panicker;
+
+ private final String tsoHostAndPort;
+
+ private final TSOStateManager stateManager;
+ private final ExecutorService tsoStateInitializer = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder()
+ .setNameFormat("tso-state-initializer")
+ .setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ panicker.panic(t + " threw exception", e);
+ }
+ })
+ .build());
+
+
+ private final long leasePeriodInMs;
+ private final TSOChannelHandler tsoChannelHandler;
+ private int leaseNodeVersion;
+ private final AtomicLong endLeaseInMs = new AtomicLong(0L);
+ private final AtomicLong baseTimeInMs = new AtomicLong(0L);
+
+ private final String leasePath;
+ private final String currentTSOPath;
+
+ LeaseManager(String tsoHostAndPort,
+ TSOChannelHandler tsoChannelHandler,
+ TSOStateManager stateManager,
+ long leasePeriodInMs,
+ String leasePath,
+ String currentTSOPath,
+ CuratorFramework zkClient,
+ Panicker panicker) {
+
+ this.tsoHostAndPort = tsoHostAndPort;
+ this.tsoChannelHandler = tsoChannelHandler;
+ this.stateManager = stateManager;
+ this.leasePeriodInMs = leasePeriodInMs;
+ this.leasePath = leasePath;
+ this.currentTSOPath = currentTSOPath;
+ this.zkClient = zkClient;
+ this.panicker = panicker;
+ LOG.info("LeaseManager {} initialized. Lease period {}ms", toString(), leasePeriodInMs);
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // LeaseManagement implementation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public void startService() throws LeaseManagementException {
+ createLeaseManagementZNode();
+ createCurrentTSOZNode();
+ startAndWait();
+ }
+
+ @Override
+ public void stopService() throws LeaseManagementException {
+ stopAndWait();
+ }
+
+ @Override
+ public boolean stillInLeasePeriod() {
+ return System.currentTimeMillis() <= getEndLeaseInMs();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // End LeaseManagement implementation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ void tryToGetInitialLeasePeriod() throws Exception {
+ baseTimeInMs.set(System.currentTimeMillis());
+ if (canAcquireLease()) {
+ endLeaseInMs.set(baseTimeInMs.get() + leasePeriodInMs);
+ LOG.info("{} got the lease (Master) Ver. {}/End of lease: {}ms", tsoHostAndPort,
+ leaseNodeVersion, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(endLeaseInMs));
+ tsoStateInitializer.submit(new Runnable() {
+ // TSO State initialization
+ @Override
+ public void run() {
+ try {
+ TSOState newTSOState = stateManager.reset();
+ advertiseTSOServerInfoThroughZK(newTSOState.getEpoch());
+ tsoChannelHandler.reconnect();
+ } catch (Exception e) {
+ Thread t = Thread.currentThread();
+ t.getUncaughtExceptionHandler().uncaughtException(t, e);
+ }
+ }
+ });
+ } else {
+ tsoStateInitializer.submit(new Runnable() {
+ // TSO State initialization
+ @Override
+ public void run() {
+ // In case the TSO was paused close the connection
+ tsoChannelHandler.closeConnection();
+ }
+ });
+ }
+ }
+
+ void tryToRenewLeasePeriod() throws Exception {
+ baseTimeInMs.set(System.currentTimeMillis());
+ if (canAcquireLease()) {
+ if (System.currentTimeMillis() > getEndLeaseInMs()) {
+ endLeaseInMs.set(0L);
+ LOG.warn("{} expired lease! Releasing lease to start Master re-election", tsoHostAndPort);
+ tsoChannelHandler.closeConnection();
+ } else {
+ endLeaseInMs.set(baseTimeInMs.get() + leasePeriodInMs);
+ LOG.trace("{} renewed lease: Version {}/End of lease at {}ms",
+ tsoHostAndPort, leaseNodeVersion, endLeaseInMs);
+ }
+ } else {
+ endLeaseInMs.set(0L);
+ LOG.warn("{} lost the lease (Ver. {})! Other instance is now Master",
+ tsoHostAndPort, leaseNodeVersion);
+ tsoChannelHandler.closeConnection();
+ }
+ }
+
+ private boolean haveLease() {
+ return stillInLeasePeriod();
+ }
+
+ private long getEndLeaseInMs() {
+ return endLeaseInMs.get();
+ }
+
+ private boolean canAcquireLease() throws Exception {
+ try {
+ int previousLeaseNodeVersion = leaseNodeVersion;
+ final byte[] instanceInfo = tsoHostAndPort.getBytes(Charsets.UTF_8);
+ // Try to acquire the lease
+ Stat stat = zkClient.setData().withVersion(previousLeaseNodeVersion)
+ .forPath(leasePath, instanceInfo);
+ leaseNodeVersion = stat.getVersion();
+ LOG.trace("{} got new lease version {}", tsoHostAndPort, leaseNodeVersion);
+ } catch (KeeperException.BadVersionException e) {
+ return false;
+ }
+ return true;
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // AbstractScheduledService implementation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ protected void startUp() {
+ }
+
+ @Override
+ protected void shutDown() {
+ try {
+ tsoChannelHandler.close();
+ LOG.info("Channel handler closed");
+ } catch (IOException e) {
+ LOG.error("Error closing TSOChannelHandler", e);
+ }
+ }
+
+ @Override
+ protected void runOneIteration() throws Exception {
+
+ if (!haveLease()) {
+ tryToGetInitialLeasePeriod();
+ } else {
+ tryToRenewLeasePeriod();
+ }
+
+ }
+
+ @Override
+ protected Scheduler scheduler() {
+
+ final long guardLeasePeriodInMs = leasePeriodInMs / 4;
+
+ return new AbstractScheduledService.CustomScheduler() {
+
+ @Override
+ protected Schedule getNextSchedule() throws Exception {
+ if (!haveLease()) {
+ // Get the current node version...
+ Stat stat = zkClient.checkExists().forPath(leasePath);
+ leaseNodeVersion = stat.getVersion();
+ LOG.trace("{} will try to get lease (with Ver. {}) in {}ms", tsoHostAndPort, leaseNodeVersion,
+ leasePeriodInMs);
+ // ...and wait the lease period
+ return new Schedule(leasePeriodInMs, TimeUnit.MILLISECONDS);
+ } else {
+ long waitTimeInMs = getEndLeaseInMs() - System.currentTimeMillis() - guardLeasePeriodInMs;
+ LOG.trace("{} will try to renew lease (with Ver. {}) in {}ms", tsoHostAndPort,
+ leaseNodeVersion, waitTimeInMs);
+ return new Schedule(waitTimeInMs, TimeUnit.MILLISECONDS);
+ }
+ }
+ };
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public String toString() {
+ return tsoHostAndPort;
+ }
+
+ private void createLeaseManagementZNode() throws LeaseManagementException {
+ try {
+ validateZKPath(leasePath);
+ } catch (Exception e) {
+ throw new LeaseManagementException("Error creating Lease Management ZNode", e);
+ }
+ }
+
+ private void createCurrentTSOZNode() throws LeaseManagementException {
+ try {
+ validateZKPath(currentTSOPath);
+ } catch (Exception e) {
+ throw new LeaseManagementException("Error creating TSO ZNode", e);
+ }
+ }
+
+ private void validateZKPath(String zkPath) throws Exception {
+ EnsurePath path = zkClient.newNamespaceAwareEnsurePath(zkPath);
+ path.ensure(zkClient.getZookeeperClient());
+ Stat stat = zkClient.checkExists().forPath(zkPath);
+ Preconditions.checkNotNull(stat);
+ LOG.info("Path {} ensured", path.getPath());
+ }
+
+ private void advertiseTSOServerInfoThroughZK(long epoch) throws Exception {
+
+ Stat previousTSOZNodeStat = new Stat();
+ byte[] previousTSOInfoAsBytes = zkClient.getData().storingStatIn(previousTSOZNodeStat).forPath(currentTSOPath);
+ if (previousTSOInfoAsBytes != null && !new String(previousTSOInfoAsBytes, Charsets.UTF_8).isEmpty()) {
+ String previousTSOInfo = new String(previousTSOInfoAsBytes, Charsets.UTF_8);
+ String[] previousTSOAndEpochArray = previousTSOInfo.split("#");
+ Preconditions.checkArgument(previousTSOAndEpochArray.length == 2, "Incorrect TSO Info found: ", previousTSOInfo);
+ long oldEpoch = Long.parseLong(previousTSOAndEpochArray[1]);
+ if (oldEpoch > epoch) {
+ throw new LeaseManagementException("Another TSO replica was found " + previousTSOInfo);
+ }
+ }
+ String tsoInfoAsString = tsoHostAndPort + "#" + Long.toString(epoch);
+ byte[] tsoInfoAsBytes = tsoInfoAsString.getBytes(Charsets.UTF_8);
+ zkClient.setData().withVersion(previousTSOZNodeStat.getVersion()).forPath(currentTSOPath, tsoInfoAsBytes);
+ LOG.info("TSO instance {} (Epoch {}) advertised through ZK", tsoHostAndPort, epoch);
+
+ }
+
+}
[51/52] [abbrv] incubator-omid git commit: Merge pull request #92
from yahoo/code-quality-reports
Posted by ik...@apache.org.
Merge pull request #92 from yahoo/code-quality-reports
Add code quality reports: checkstyle and PMD
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/0c0bcf5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/0c0bcf5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/0c0bcf5b
Branch: refs/heads/master
Commit: 0c0bcf5bfa16859447e5554e7fad579f0f43b32c
Parents: 410653e 4f3e4c2
Author: ikatkov <ik...@gmail.com>
Authored: Wed Apr 20 16:42:43 2016 -0700
Committer: ikatkov <ik...@gmail.com>
Committed: Wed Apr 20 16:42:43 2016 -0700
----------------------------------------------------------------------
doc/site/markdown/coding-guide-and-style.md | 10 +-
misc/omid_checks.xml | 211 +++++++++++++++++++++++
pom.xml | 37 ++++
3 files changed, 255 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[05/52] [abbrv] incubator-omid git commit: Merge pull request #87
from yahoo/pr-headers
Posted by ik...@apache.org.
Merge pull request #87 from yahoo/pr-headers
Updated source code with Apache license header
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/a02d74a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/a02d74a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/a02d74a7
Branch: refs/heads/master
Commit: a02d74a72e1df8b9334c713c9f763649849403a3
Parents: a5b4f42 0f13d2a
Author: Francisco Pérez-Sorrosal <fp...@gmail.com>
Authored: Wed Apr 13 13:47:03 2016 -0700
Committer: Francisco Pérez-Sorrosal <fp...@gmail.com>
Committed: Wed Apr 13 13:47:03 2016 -0700
----------------------------------------------------------------------
.../benchmarks/hbase/HBaseCommitTableTester.java | 14 ++++++++------
.../yahoo/omid/benchmarks/tso/RawTxRunner.java | 14 ++++++++------
.../omid/benchmarks/tso/TSOServerBenchmark.java | 14 ++++++++------
.../benchmarks/tso/TSOServerBenchmarkConfig.java | 14 ++++++++------
.../omid/benchmarks/utils/UniformGenerator.java | 14 ++++++++------
.../omid/metrics/CodahaleMetricsConfig.java | 14 ++++++++------
.../omid/metrics/CodahaleMetricsProvider.java | 14 ++++++++------
.../com/yahoo/omid/committable/CommitTable.java | 14 ++++++++------
.../omid/committable/InMemoryCommitTable.java | 14 ++++++++------
.../yahoo/omid/committable/NullCommitTable.java | 14 ++++++++------
.../omid/committable/NullCommitTableTest.java | 17 +++++++++++++++++
.../src/main/java/com/yahoo/omid/YAMLUtils.java | 14 ++++++++------
.../src/main/java/com/yahoo/omid/zk/ZKUtils.java | 14 ++++++++------
common/src/main/proto/TSOProto.proto | 14 ++++++++------
.../test/java/com/yahoo/omid/YAMLUtilsTest.java | 17 +++++++++++++++++
.../com/yahoo/omid/examples/BasicExample.java | 14 ++++++++------
.../omid/examples/ConfigurationExample.java | 14 ++++++++------
.../omid/examples/SnapshotIsolationExample.java | 14 ++++++++------
.../yahoo/omid/transaction/ColumnWrapper.java | 14 ++++++++------
.../transaction/HBaseAsyncPostCommitter.java | 14 ++++++++------
.../com/yahoo/omid/transaction/HBaseCellId.java | 14 ++++++++------
.../HBaseOmidClientConfiguration.java | 14 ++++++++------
.../omid/transaction/HBaseSyncPostCommitter.java | 14 ++++++++------
.../yahoo/omid/transaction/HBaseTransaction.java | 14 ++++++++------
.../omid/transaction/HBaseTransactionClient.java | 14 ++++++++------
.../transaction/HBaseTransactionManager.java | 14 ++++++++------
.../java/com/yahoo/omid/transaction/TTable.java | 14 ++++++++------
.../transaction/TTableCellGetterAdapter.java | 14 ++++++++------
.../com/yahoo/omid/transaction/OmidTestBase.java | 17 +++++++++++++++++
.../TestAsynchronousPostCommitter.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestAutoFlush.java | 17 +++++++++++++++++
.../omid/transaction/TestBasicTransaction.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestCellUtils.java | 17 +++++++++++++++++
.../omid/transaction/TestColumnIterator.java | 17 +++++++++++++++++
.../com/yahoo/omid/transaction/TestDeletion.java | 17 +++++++++++++++++
.../transaction/TestEndToEndScenariosWithHA.java | 17 +++++++++++++++++
.../com/yahoo/omid/transaction/TestFilters.java | 17 +++++++++++++++++
.../transaction/TestHALeaseManagementModule.java | 14 ++++++++------
.../TestHBaseOmidClientConfiguration.java | 17 +++++++++++++++++
.../transaction/TestHBaseTransactionClient.java | 17 +++++++++++++++++
.../transaction/TestHBaseTransactionManager.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestMultiplePut.java | 17 +++++++++++++++++
.../com/yahoo/omid/transaction/TestReadPath.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestShadowCells.java | 17 +++++++++++++++++
.../omid/transaction/TestSingleColumnFamily.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestTSOModule.java | 17 +++++++++++++++++
.../omid/transaction/TestTTableBehaviour.java | 17 +++++++++++++++++
.../omid/transaction/TestTransactionCleanup.java | 17 +++++++++++++++++
.../transaction/TestTransactionConflict.java | 17 +++++++++++++++++
.../omid/transaction/TestTxMgrFailover.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestUpdateScan.java | 17 +++++++++++++++++
.../DefaultHBaseCommitTableStorageModule.java | 14 ++++++++------
.../omid/committable/hbase/HBaseCommitTable.java | 14 ++++++++------
.../hbase/HBaseCommitTableConfig.java | 14 ++++++++------
.../hbase/HBaseCommitTableStorageModule.java | 14 ++++++++------
.../committable/hbase/TestHBaseCommitTable.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/HBaseConfigModule.java | 14 ++++++++------
.../omid/committable/hbase/KeyGenerator.java | 14 ++++++++------
.../hbase/KeyGeneratorImplementations.java | 14 ++++++++------
.../com/yahoo/omid/tools/hbase/HBaseLogin.java | 14 ++++++++------
.../omid/tools/hbase/SecureHBaseConfig.java | 14 ++++++++------
.../com/yahoo/omid/transaction/CellInfo.java | 14 ++++++++------
.../com/yahoo/omid/transaction/CellUtils.java | 14 ++++++++------
.../hbase/TestHBaseCommitTableKey.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/CompactorUtil.java | 14 ++++++++------
.../transaction/HRegionCellGetterAdapter.java | 14 ++++++++------
.../yahoo/omid/transaction/OmidCompactor.java | 14 ++++++++------
.../hbase/regionserver/CompactorScanner.java | 14 ++++++++------
.../TSOForHBaseCompactorTestModule.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestCompaction.java | 17 +++++++++++++++++
.../omid/transaction/TestCompactorScanner.java | 17 +++++++++++++++++
hbase-coprocessor/src/test/resources/log4j.xml | 19 +++++++++++++++++++
.../src/main/java/com/yahoo/omid/HBaseShims.java | 14 ++++++++------
.../apache/hadoop/hbase/regionserver/Region.java | 14 ++++++++------
.../hbase/regionserver/ScannerContext.java | 14 ++++++++------
.../src/main/java/com/yahoo/omid/HBaseShims.java | 14 ++++++++------
.../yahoo/omid/tools/hbase/OmidTableManager.java | 14 ++++++++------
.../omid/tools/hbase/TestOmidTableManager.java | 17 +++++++++++++++++
.../omid/metrics/AbstractMetricsConfig.java | 14 ++++++++------
.../java/com/yahoo/omid/metrics/Counter.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Gauge.java | 14 ++++++++------
.../java/com/yahoo/omid/metrics/Histogram.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Meter.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Metric.java | 14 ++++++++------
.../com/yahoo/omid/metrics/MetricsProvider.java | 14 ++++++++------
.../com/yahoo/omid/metrics/MetricsRegistry.java | 14 ++++++++------
.../yahoo/omid/metrics/MetricsRegistryMap.java | 14 ++++++++------
.../com/yahoo/omid/metrics/MetricsUtils.java | 14 ++++++++------
.../yahoo/omid/metrics/NullMetricsProvider.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Timer.java | 14 ++++++++------
misc/header.txt | 14 ++++++++------
pom.xml | 12 +-----------
.../com/yahoo/statemachine/StateMachine.java | 14 ++++++++------
.../statemachine/StateMachineLogParser.java | 14 ++++++++------
.../com/yahoo/statemachine/TestStateMachine.java | 17 +++++++++++++++++
.../DefaultHBaseTimestampStorageModule.java | 14 ++++++++------
.../storage/DefaultZKTimestampStorageModule.java | 14 ++++++++------
.../timestamp/storage/HBaseTimestampStorage.java | 14 ++++++++------
.../storage/HBaseTimestampStorageConfig.java | 14 ++++++++------
.../storage/HBaseTimestampStorageModule.java | 14 ++++++++------
.../omid/timestamp/storage/TimestampStorage.java | 14 ++++++++------
.../yahoo/omid/timestamp/storage/ZKModule.java | 14 ++++++++------
.../omid/timestamp/storage/ZKTimestampPaths.java | 14 ++++++++------
.../timestamp/storage/ZKTimestampStorage.java | 14 ++++++++------
.../storage/ZKTimestampStorageModule.java | 14 ++++++++------
.../storage/TestHBaseTimestampStorage.java | 17 +++++++++++++++++
.../storage/TestZKTimestampStorage.java | 17 +++++++++++++++++
.../omid/transaction/AbstractTransaction.java | 14 ++++++++------
.../transaction/AbstractTransactionManager.java | 14 ++++++++------
.../omid/transaction/CommitTimestampLocator.java | 14 ++++++++------
.../omid/transaction/PostCommitActions.java | 14 ++++++++------
.../omid/transaction/RollbackException.java | 14 ++++++++------
.../com/yahoo/omid/transaction/Transaction.java | 14 ++++++++------
.../omid/transaction/TransactionException.java | 14 ++++++++------
.../omid/transaction/TransactionManager.java | 14 ++++++++------
.../transaction/TransactionManagerException.java | 14 ++++++++------
.../com/yahoo/omid/tso/util/DummyCellIdImpl.java | 14 ++++++++------
.../com/yahoo/omid/tsoclient/AbortException.java | 14 ++++++++------
.../java/com/yahoo/omid/tsoclient/CellId.java | 14 ++++++++------
.../yahoo/omid/tsoclient/ClosingException.java | 14 ++++++++------
.../omid/tsoclient/ConnectionException.java | 14 ++++++++------
.../omid/tsoclient/ForwardingTSOFuture.java | 14 ++++++++------
.../omid/tsoclient/HandshakeFailedException.java | 14 ++++++++------
.../com/yahoo/omid/tsoclient/MockTSOClient.java | 14 ++++++++------
.../yahoo/omid/tsoclient/NewTSOException.java | 14 ++++++++------
.../omid/tsoclient/OmidClientConfiguration.java | 14 ++++++++------
.../tsoclient/ServiceUnavailableException.java | 14 ++++++++------
.../java/com/yahoo/omid/tsoclient/TSOClient.java | 14 ++++++++------
.../java/com/yahoo/omid/tsoclient/TSOFuture.java | 14 ++++++++------
.../com/yahoo/omid/tsoclient/TSOProtocol.java | 14 ++++++++------
.../yahoo/omid/tsoclient/TestMockTSOClient.java | 17 +++++++++++++++++
.../tsoclient/TestOmidClientConfiguration.java | 17 +++++++++++++++++
.../src/main/java/com/yahoo/omid/tso/Cache.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/CacheEvaluation.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/CommitHashMap.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/DisruptorModule.java | 14 ++++++++------
.../yahoo/omid/tso/FatalExceptionHandler.java | 14 ++++++++------
.../yahoo/omid/tso/HALeaseManagementModule.java | 14 ++++++++------
.../tso/InMemoryCommitTableStorageModule.java | 14 ++++++++------
.../omid/tso/InMemoryTimestampStorageModule.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/LeaseManagement.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/LeaseManager.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/LongCache.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/MockPanicker.java | 14 ++++++++------
.../com/yahoo/omid/tso/MonitoringContext.java | 14 ++++++++------
.../yahoo/omid/tso/NetworkInterfaceUtils.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/Panicker.java | 14 ++++++++------
.../yahoo/omid/tso/PausableTimestampOracle.java | 14 ++++++++------
.../com/yahoo/omid/tso/PersistenceProcessor.java | 14 ++++++++------
.../yahoo/omid/tso/PersistenceProcessorImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/ReplyProcessor.java | 14 ++++++++------
.../com/yahoo/omid/tso/ReplyProcessorImpl.java | 14 ++++++++------
.../com/yahoo/omid/tso/RequestProcessor.java | 14 ++++++++------
.../com/yahoo/omid/tso/RequestProcessorImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/RetryProcessor.java | 14 ++++++++------
.../com/yahoo/omid/tso/RetryProcessorImpl.java | 14 ++++++++------
.../com/yahoo/omid/tso/SystemExitPanicker.java | 14 ++++++++------
.../com/yahoo/omid/tso/TSOChannelHandler.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/TSOModule.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/TSOServer.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TSOServerConfig.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TSOStateManager.java | 14 ++++++++------
.../com/yahoo/omid/tso/TSOStateManagerImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TimestampOracle.java | 14 ++++++++------
.../com/yahoo/omid/tso/TimestampOracleImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TsoServerDaemon.java | 14 ++++++++------
.../omid/tso/VoidLeaseManagementModule.java | 14 ++++++++------
.../com/yahoo/omid/tso/VoidLeaseManager.java | 14 ++++++++------
.../src/test/java/com/yahoo/omid/TestUtils.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/PausableLeaseManager.java | 17 +++++++++++++++++
.../yahoo/omid/tso/ProgrammableTSOServer.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/tso/TSOMockModule.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TSOServerConfigTest.java | 17 +++++++++++++++++
.../test/java/com/yahoo/omid/tso/TestBatch.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestLeaseManager.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/tso/TestLongCache.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/tso/TestPanicker.java | 17 +++++++++++++++++
.../yahoo/omid/tso/TestPersistenceProcessor.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestRequestProcessor.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestRetryProcessor.java | 17 +++++++++++++++++
.../omid/tso/TestTSOChannelHandlerNetty.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestTSOStateManager.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestTimestampOracle.java | 17 +++++++++++++++++
.../yahoo/omid/tsoclient/TSOClientAccessor.java | 17 +++++++++++++++++
.../yahoo/omid/tsoclient/TSOClientOneShot.java | 17 +++++++++++++++++
.../com/yahoo/omid/tsoclient/TSOClientRaw.java | 17 +++++++++++++++++
...ationOfTSOClientServerBasicFunctionality.java | 17 +++++++++++++++++
.../tsoclient/TestTSOClientConnectionToTSO.java | 17 +++++++++++++++++
...estTSOClientRequestAndResponseBehaviours.java | 17 +++++++++++++++++
.../tsoclient/TestTSOClientResponseHandling.java | 17 +++++++++++++++++
.../omid/tsoclient/TestUnconnectedTSOClient.java | 17 +++++++++++++++++
191 files changed, 2054 insertions(+), 797 deletions(-)
----------------------------------------------------------------------
[22/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java b/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
deleted file mode 100644
index e6ff7f1..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/LeaseManager.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.AbstractScheduledService;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.tso.TSOStateManager.TSOState;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.utils.EnsurePath;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Encompasses all the required elements to control the leases required for
- * identifying the master instance when running multiple TSO instances for HA
- * It delegates the initialization of the TSO state and the publication of
- * the instance information when getting the lease to an asynchronous task to
- * continue managing the leases without interruptions.
- */
-class LeaseManager extends AbstractScheduledService implements LeaseManagement {
-
- private static final Logger LOG = LoggerFactory.getLogger(LeaseManager.class);
-
- private final CuratorFramework zkClient;
-
- private final Panicker panicker;
-
- private final String tsoHostAndPort;
-
- private final TSOStateManager stateManager;
- private final ExecutorService tsoStateInitializer = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder()
- .setNameFormat("tso-state-initializer")
- .setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- panicker.panic(t + " threw exception", e);
- }
- })
- .build());
-
-
- private final long leasePeriodInMs;
- private final TSOChannelHandler tsoChannelHandler;
- private int leaseNodeVersion;
- private final AtomicLong endLeaseInMs = new AtomicLong(0L);
- private final AtomicLong baseTimeInMs = new AtomicLong(0L);
-
- private final String leasePath;
- private final String currentTSOPath;
-
- LeaseManager(String tsoHostAndPort,
- TSOChannelHandler tsoChannelHandler,
- TSOStateManager stateManager,
- long leasePeriodInMs,
- String leasePath,
- String currentTSOPath,
- CuratorFramework zkClient,
- Panicker panicker) {
-
- this.tsoHostAndPort = tsoHostAndPort;
- this.tsoChannelHandler = tsoChannelHandler;
- this.stateManager = stateManager;
- this.leasePeriodInMs = leasePeriodInMs;
- this.leasePath = leasePath;
- this.currentTSOPath = currentTSOPath;
- this.zkClient = zkClient;
- this.panicker = panicker;
- LOG.info("LeaseManager {} initialized. Lease period {}ms", toString(), leasePeriodInMs);
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // LeaseManagement implementation
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public void startService() throws LeaseManagementException {
- createLeaseManagementZNode();
- createCurrentTSOZNode();
- startAndWait();
- }
-
- @Override
- public void stopService() throws LeaseManagementException {
- stopAndWait();
- }
-
- @Override
- public boolean stillInLeasePeriod() {
- return System.currentTimeMillis() <= getEndLeaseInMs();
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // End LeaseManagement implementation
- // ----------------------------------------------------------------------------------------------------------------
-
- void tryToGetInitialLeasePeriod() throws Exception {
- baseTimeInMs.set(System.currentTimeMillis());
- if (canAcquireLease()) {
- endLeaseInMs.set(baseTimeInMs.get() + leasePeriodInMs);
- LOG.info("{} got the lease (Master) Ver. {}/End of lease: {}ms", tsoHostAndPort,
- leaseNodeVersion, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(endLeaseInMs));
- tsoStateInitializer.submit(new Runnable() {
- // TSO State initialization
- @Override
- public void run() {
- try {
- TSOState newTSOState = stateManager.reset();
- advertiseTSOServerInfoThroughZK(newTSOState.getEpoch());
- tsoChannelHandler.reconnect();
- } catch (Exception e) {
- Thread t = Thread.currentThread();
- t.getUncaughtExceptionHandler().uncaughtException(t, e);
- }
- }
- });
- } else {
- tsoStateInitializer.submit(new Runnable() {
- // TSO State initialization
- @Override
- public void run() {
- // In case the TSO was paused close the connection
- tsoChannelHandler.closeConnection();
- }
- });
- }
- }
-
- void tryToRenewLeasePeriod() throws Exception {
- baseTimeInMs.set(System.currentTimeMillis());
- if (canAcquireLease()) {
- if (System.currentTimeMillis() > getEndLeaseInMs()) {
- endLeaseInMs.set(0L);
- LOG.warn("{} expired lease! Releasing lease to start Master re-election", tsoHostAndPort);
- tsoChannelHandler.closeConnection();
- } else {
- endLeaseInMs.set(baseTimeInMs.get() + leasePeriodInMs);
- LOG.trace("{} renewed lease: Version {}/End of lease at {}ms",
- tsoHostAndPort, leaseNodeVersion, endLeaseInMs);
- }
- } else {
- endLeaseInMs.set(0L);
- LOG.warn("{} lost the lease (Ver. {})! Other instance is now Master",
- tsoHostAndPort, leaseNodeVersion);
- tsoChannelHandler.closeConnection();
- }
- }
-
- private boolean haveLease() {
- return stillInLeasePeriod();
- }
-
- private long getEndLeaseInMs() {
- return endLeaseInMs.get();
- }
-
- private boolean canAcquireLease() throws Exception {
- try {
- int previousLeaseNodeVersion = leaseNodeVersion;
- final byte[] instanceInfo = tsoHostAndPort.getBytes(Charsets.UTF_8);
- // Try to acquire the lease
- Stat stat = zkClient.setData().withVersion(previousLeaseNodeVersion)
- .forPath(leasePath, instanceInfo);
- leaseNodeVersion = stat.getVersion();
- LOG.trace("{} got new lease version {}", tsoHostAndPort, leaseNodeVersion);
- } catch (KeeperException.BadVersionException e) {
- return false;
- }
- return true;
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // AbstractScheduledService implementation
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- protected void startUp() {
- }
-
- @Override
- protected void shutDown() {
- try {
- tsoChannelHandler.close();
- LOG.info("Channel handler closed");
- } catch (IOException e) {
- LOG.error("Error closing TSOChannelHandler", e);
- }
- }
-
- @Override
- protected void runOneIteration() throws Exception {
-
- if (!haveLease()) {
- tryToGetInitialLeasePeriod();
- } else {
- tryToRenewLeasePeriod();
- }
-
- }
-
- @Override
- protected Scheduler scheduler() {
-
- final long guardLeasePeriodInMs = leasePeriodInMs / 4;
-
- return new AbstractScheduledService.CustomScheduler() {
-
- @Override
- protected Schedule getNextSchedule() throws Exception {
- if (!haveLease()) {
- // Get the current node version...
- Stat stat = zkClient.checkExists().forPath(leasePath);
- leaseNodeVersion = stat.getVersion();
- LOG.trace("{} will try to get lease (with Ver. {}) in {}ms", tsoHostAndPort, leaseNodeVersion,
- leasePeriodInMs);
- // ...and wait the lease period
- return new Schedule(leasePeriodInMs, TimeUnit.MILLISECONDS);
- } else {
- long waitTimeInMs = getEndLeaseInMs() - System.currentTimeMillis() - guardLeasePeriodInMs;
- LOG.trace("{} will try to renew lease (with Ver. {}) in {}ms", tsoHostAndPort,
- leaseNodeVersion, waitTimeInMs);
- return new Schedule(waitTimeInMs, TimeUnit.MILLISECONDS);
- }
- }
- };
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public String toString() {
- return tsoHostAndPort;
- }
-
- private void createLeaseManagementZNode() throws LeaseManagementException {
- try {
- validateZKPath(leasePath);
- } catch (Exception e) {
- throw new LeaseManagementException("Error creating Lease Management ZNode", e);
- }
- }
-
- private void createCurrentTSOZNode() throws LeaseManagementException {
- try {
- validateZKPath(currentTSOPath);
- } catch (Exception e) {
- throw new LeaseManagementException("Error creating TSO ZNode", e);
- }
- }
-
- private void validateZKPath(String zkPath) throws Exception {
- EnsurePath path = zkClient.newNamespaceAwareEnsurePath(zkPath);
- path.ensure(zkClient.getZookeeperClient());
- Stat stat = zkClient.checkExists().forPath(zkPath);
- Preconditions.checkNotNull(stat);
- LOG.info("Path {} ensured", path.getPath());
- }
-
- private void advertiseTSOServerInfoThroughZK(long epoch) throws Exception {
-
- Stat previousTSOZNodeStat = new Stat();
- byte[] previousTSOInfoAsBytes = zkClient.getData().storingStatIn(previousTSOZNodeStat).forPath(currentTSOPath);
- if (previousTSOInfoAsBytes != null && !new String(previousTSOInfoAsBytes, Charsets.UTF_8).isEmpty()) {
- String previousTSOInfo = new String(previousTSOInfoAsBytes, Charsets.UTF_8);
- String[] previousTSOAndEpochArray = previousTSOInfo.split("#");
- Preconditions.checkArgument(previousTSOAndEpochArray.length == 2, "Incorrect TSO Info found: ", previousTSOInfo);
- long oldEpoch = Long.parseLong(previousTSOAndEpochArray[1]);
- if (oldEpoch > epoch) {
- throw new LeaseManagementException("Another TSO replica was found " + previousTSOInfo);
- }
- }
- String tsoInfoAsString = tsoHostAndPort + "#" + Long.toString(epoch);
- byte[] tsoInfoAsBytes = tsoInfoAsString.getBytes(Charsets.UTF_8);
- zkClient.setData().withVersion(previousTSOZNodeStat.getVersion()).forPath(currentTSOPath, tsoInfoAsBytes);
- LOG.info("TSO instance {} (Epoch {}) advertised through ZK", tsoHostAndPort, epoch);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java b/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
deleted file mode 100644
index e140d50..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/LongCache.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import java.util.Arrays;
-
-public class LongCache implements Cache {
-
- public static final long RESET_VALUE = 0L;
-
- private final long[] cache;
- private final int size;
- private final int associativity;
-
- public LongCache(int size, int associativity) {
- this.size = size;
- this.cache = new long[2 * (size + associativity)];
- this.associativity = associativity;
- }
-
- @Override
- public void reset() {
- Arrays.fill(cache, RESET_VALUE);
- }
-
- /* (non-Javadoc)
- * @see com.yahoo.omid.tso.Cache#set(long, long)
- */
- @Override
- public long set(long key, long value) {
- final int index = index(key);
- int oldestIndex = 0;
- long oldestValue = Long.MAX_VALUE;
- for (int i = 0; i < associativity; ++i) {
- int currIndex = 2 * (index + i);
- if (cache[currIndex] == key) {
- oldestValue = 0;
- oldestIndex = currIndex;
- break;
- }
- if (cache[currIndex + 1] <= oldestValue) {
- oldestValue = cache[currIndex + 1];
- oldestIndex = currIndex;
- }
- }
- cache[oldestIndex] = key;
- cache[oldestIndex + 1] = value;
- return oldestValue;
- }
-
- /* (non-Javadoc)
- * @see com.yahoo.omid.tso.Cache#get(long)
- */
- @Override
- public long get(long key) {
- final int index = index(key);
- for (int i = 0; i < associativity; ++i) {
- int currIndex = 2 * (index + i);
- if (cache[currIndex] == key) {
- return cache[currIndex + 1];
- }
- }
- return 0;
- }
-
- private int index(long hash) {
- return (int) (Math.abs(hash) % size);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java b/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
deleted file mode 100644
index aec6141..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/MockPanicker.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MockPanicker implements Panicker {
- private static final Logger LOG = LoggerFactory.getLogger(MockPanicker.class);
-
- @Override
- public void panic(String reason, Throwable cause) {
- LOG.error("PANICKING: {}", reason, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java b/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
deleted file mode 100644
index c52e96a..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/MonitoringContext.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.base.Stopwatch;
-import com.google.common.base.Throwables;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.NotThreadSafe;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-
-@NotThreadSafe
-public class MonitoringContext {
-
- private static final Logger LOG = LoggerFactory.getLogger(MonitoringContext.class);
-
- private volatile boolean flag;
- private Map<String, Long> elapsedTimeMsMap = new HashMap<>();
- private Map<String, Stopwatch> timers = new ConcurrentHashMap<>();
- private MetricsRegistry metrics;
-
- public MonitoringContext(MetricsRegistry metrics) {
- this.metrics = metrics;
- }
-
- public void timerStart(String name) {
- Stopwatch stopwatch = new Stopwatch();
- stopwatch.start();
- timers.put(name, stopwatch);
- }
-
- public void timerStop(String name) {
- if (flag) {
- LOG.warn("timerStop({}) called after publish. Measurement was ignored. {}", name, Throwables.getStackTraceAsString(new Exception()));
- return;
- }
- Stopwatch activeStopwatch = timers.get(name);
- if (activeStopwatch == null) {
- throw new IllegalStateException(
- String.format("There is no %s timer in the %s monitoring context.", name, this));
- }
- activeStopwatch.stop();
- elapsedTimeMsMap.put(name, activeStopwatch.elapsedTime(TimeUnit.NANOSECONDS));
- timers.remove(name);
- }
-
- public void publish() {
- flag = true;
- for (String name : elapsedTimeMsMap.keySet()) {
- Long durationInNs = elapsedTimeMsMap.get(name);
- metrics.timer(name("tso", name)).update(durationInNs);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java b/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
deleted file mode 100644
index 7a32f1e..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/NetworkInterfaceUtils.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.net.HostAndPort;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-import java.util.Enumeration;
-
-final public class NetworkInterfaceUtils {
-
- private static final Logger LOG = LoggerFactory.getLogger(NetworkInterfaceUtils.class);
-
- /**
- * Returns an <code>InetAddress</code> object encapsulating what is most
- * likely the machine's LAN IP address.
- * <p/>
- * This method is intended for use as a replacement of JDK method
- * <code>InetAddress.getLocalHost</code>, because that method is ambiguous
- * on Linux systems. Linux systems enumerate the loopback network
- * interface the same way as regular LAN network interfaces, but the JDK
- * <code>InetAddress.getLocalHost</code> method does not specify the
- * algorithm used to select the address returned under such circumstances,
- * and will often return the loopback address, which is not valid for
- * network communication. Details
- * <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4665037">here</a>.
- * <p/>
- * This method will scan all IP addresses on a particular network interface
- * specified as parameter on the host machine to determine the IP address
- * most likely to be the machine's LAN address. If the machine has multiple
- * IP addresses, this method will prefer a site-local IP address (e.g.
- * 192.168.x.x or 10.10.x.x, usually IPv4) if the machine has one (and will
- * return the first site-local address if the machine has more than one),
- * but if the machine does not hold a site-local address, this method will
- * return simply the first non-loopback address found (IPv4 or IPv6).
- * <p/>
- * If this method cannot find a non-loopback address using this selection
- * algorithm, it will fall back to calling and returning the result of JDK
- * method <code>InetAddress.getLocalHost()</code>.
- * <p/>
- *
- * @param ifaceName
- * The name of the network interface to extract the IP address
- * from
- * @throws UnknownHostException
- * If the LAN address of the machine cannot be found.
- */
- static InetAddress getIPAddressFromNetworkInterface(String ifaceName)
- throws SocketException, UnknownHostException {
-
- NetworkInterface iface = NetworkInterface.getByName(ifaceName);
- if (iface == null) {
- throw new IllegalArgumentException(
- "Network interface " + ifaceName + " not found");
- }
-
- InetAddress candidateAddress = null;
- Enumeration<InetAddress> inetAddrs = iface.getInetAddresses();
- while (inetAddrs.hasMoreElements()) {
- InetAddress inetAddr = inetAddrs.nextElement();
- if (!inetAddr.isLoopbackAddress()) {
- if (inetAddr.isSiteLocalAddress()) {
- return inetAddr; // Return non-loopback site-local address
- } else if (candidateAddress == null) {
- // Found non-loopback address, but not necessarily site-local
- candidateAddress = inetAddr;
- }
- }
- }
-
- if (candidateAddress != null) {
- // Site-local address not found, but found other non-loopback addr
- // Server might have a non-site-local address assigned to its NIC
- // (or might be running IPv6 which deprecates "site-local" concept)
- return candidateAddress;
- }
-
- // At this point, we did not find a non-loopback address.
- // Fall back to returning whatever InetAddress.getLocalHost() returns
- InetAddress jdkSuppliedAddress = InetAddress.getLocalHost();
- if (jdkSuppliedAddress == null) {
- throw new UnknownHostException(
- "InetAddress.getLocalHost() unexpectedly returned null.");
- }
- return jdkSuppliedAddress;
- }
-
- public static String getTSOHostAndPort(TSOServerConfig config) throws SocketException, UnknownHostException {
-
- // Build TSO host:port string and validate it
- final String tsoNetIfaceName = config.getNetworkIfaceName();
- InetAddress addr = getIPAddressFromNetworkInterface(tsoNetIfaceName);
- final int tsoPort = config.getPort();
-
- String tsoHostAndPortAsString = "N/A";
- try {
- tsoHostAndPortAsString = HostAndPort.fromParts(addr.getHostAddress(), tsoPort).toString();
- } catch (IllegalArgumentException e) {
- LOG.error("Cannot parse TSO host:port string {}", tsoHostAndPortAsString);
- throw e;
- }
- return tsoHostAndPortAsString;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java b/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
deleted file mode 100644
index 47860ae..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/Panicker.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-public interface Panicker {
- public void panic(String reason, Throwable cause);
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java b/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
deleted file mode 100644
index 98be866..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/PausableTimestampOracle.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-
-public class PausableTimestampOracle extends TimestampOracleImpl {
-
- private static final Logger LOG = LoggerFactory.getLogger(PausableTimestampOracle.class);
-
- private volatile boolean tsoPaused = false;
-
- @Inject
- public PausableTimestampOracle(MetricsRegistry metrics,
- TimestampStorage tsStorage,
- Panicker panicker) throws IOException {
- super(metrics, tsStorage, panicker);
- }
-
- @Override
- public long next() throws IOException {
- while (tsoPaused) {
- synchronized (this) {
- try {
- this.wait();
- } catch (InterruptedException e) {
- LOG.error("Interrupted whilst paused");
- Thread.currentThread().interrupt();
- }
- }
- }
- return super.next();
- }
-
- public synchronized void pause() {
- tsoPaused = true;
- this.notifyAll();
- }
-
- public synchronized void resume() {
- tsoPaused = false;
- this.notifyAll();
- }
-
- public boolean isTSOPaused() {
- return tsoPaused;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
deleted file mode 100644
index 4aa7b51..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessor.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.jboss.netty.channel.Channel;
-
-interface PersistenceProcessor {
- void persistCommit(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx);
-
- void persistAbort(long startTimestamp, boolean isRetry, Channel c, MonitoringContext monCtx);
-
- void persistTimestamp(long startTimestamp, Channel c, MonitoringContext monCtx);
-
- void persistLowWatermark(long lowWatermark);
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
deleted file mode 100644
index 2b46b86..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/PersistenceProcessorImpl.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.lmax.disruptor.BatchEventProcessor;
-import com.lmax.disruptor.EventFactory;
-import com.lmax.disruptor.EventHandler;
-import com.lmax.disruptor.RingBuffer;
-import com.lmax.disruptor.SequenceBarrier;
-import com.lmax.disruptor.TimeoutBlockingWaitStrategy;
-import com.lmax.disruptor.TimeoutHandler;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.Histogram;
-import com.yahoo.omid.metrics.Meter;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.Timer;
-import org.jboss.netty.channel.Channel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import javax.inject.Named;
-import java.io.IOException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-class PersistenceProcessorImpl
- implements EventHandler<PersistenceProcessorImpl.PersistEvent>, PersistenceProcessor, TimeoutHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(PersistenceProcessor.class);
-
- private final String tsoHostAndPort;
- private final LeaseManagement leaseManager;
- final ReplyProcessor reply;
- final RetryProcessor retryProc;
- final CommitTable.Client commitTableClient;
- final CommitTable.Writer writer;
- final Panicker panicker;
- final RingBuffer<PersistEvent> persistRing;
- final Batch batch;
- final Timer flushTimer;
- final Histogram batchSizeHistogram;
- final Meter timeoutMeter;
- final int batchPersistTimeoutInMs;
-
- long lastFlush = System.nanoTime();
-
- @Inject
- PersistenceProcessorImpl(TSOServerConfig config,
- MetricsRegistry metrics,
- @Named(TSO_HOST_AND_PORT_KEY) String tsoHostAndPort,
- LeaseManagement leaseManager,
- CommitTable commitTable,
- ReplyProcessor reply,
- RetryProcessor retryProc,
- Panicker panicker)
- throws IOException {
-
- this(config,
- metrics,
- tsoHostAndPort,
- new Batch(config.getMaxBatchSize()),
- leaseManager,
- commitTable,
- reply,
- retryProc,
- panicker);
-
- }
-
- @VisibleForTesting
- PersistenceProcessorImpl(TSOServerConfig config,
- MetricsRegistry metrics,
- String tsoHostAndPort,
- Batch batch,
- LeaseManagement leaseManager,
- CommitTable commitTable,
- ReplyProcessor reply,
- RetryProcessor retryProc,
- Panicker panicker)
- throws IOException {
-
- this.tsoHostAndPort = tsoHostAndPort;
- this.batch = batch;
- this.batchPersistTimeoutInMs = config.getBatchPersistTimeoutInMs();
- this.leaseManager = leaseManager;
- this.commitTableClient = commitTable.getClient();
- this.writer = commitTable.getWriter();
- this.reply = reply;
- this.retryProc = retryProc;
- this.panicker = panicker;
-
- LOG.info("Creating the persist processor with batch size {}, and timeout {}ms",
- config.getMaxBatchSize(), batchPersistTimeoutInMs);
-
- flushTimer = metrics.timer(name("tso", "persist", "flush"));
- batchSizeHistogram = metrics.histogram(name("tso", "persist", "batchsize"));
- timeoutMeter = metrics.meter(name("tso", "persist", "timeout"));
-
- // FIXME consider putting something more like a phased strategy here to avoid
- // all the syscalls
- final TimeoutBlockingWaitStrategy timeoutStrategy
- = new TimeoutBlockingWaitStrategy(config.getBatchPersistTimeoutInMs(), TimeUnit.MILLISECONDS);
-
- persistRing = RingBuffer.createSingleProducer(
- PersistEvent.EVENT_FACTORY, 1 << 20, timeoutStrategy); // 2^20 entries in ringbuffer
- SequenceBarrier persistSequenceBarrier = persistRing.newBarrier();
- BatchEventProcessor<PersistEvent> persistProcessor = new BatchEventProcessor<>(
- persistRing,
- persistSequenceBarrier,
- this);
- persistRing.addGatingSequences(persistProcessor.getSequence());
- persistProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
-
- ExecutorService persistExec = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("persist-%d").build());
- persistExec.submit(persistProcessor);
-
- }
-
- @Override
- public void onEvent(PersistEvent event, long sequence, boolean endOfBatch) throws Exception {
-
- switch (event.getType()) {
- case COMMIT:
- event.getMonCtx().timerStart("commitPersistProcessor");
- // TODO: What happens when the IOException is thrown?
- writer.addCommittedTransaction(event.getStartTimestamp(), event.getCommitTimestamp());
- batch.addCommit(event.getStartTimestamp(), event.getCommitTimestamp(), event.getChannel(),
- event.getMonCtx());
- break;
- case ABORT:
- sendAbortOrIdentifyFalsePositive(event.getStartTimestamp(), event.isRetry(), event.getChannel(),
- event.getMonCtx());
- break;
- case TIMESTAMP:
- event.getMonCtx().timerStart("timestampPersistProcessor");
- batch.addTimestamp(event.getStartTimestamp(), event.getChannel(), event.getMonCtx());
- break;
- }
- if (batch.isFull() || endOfBatch) {
- maybeFlushBatch();
- }
-
- }
-
- private void sendAbortOrIdentifyFalsePositive(long startTimestamp, boolean isRetry, Channel channel,
- MonitoringContext monCtx) {
-
- if (!isRetry) {
- reply.abortResponse(startTimestamp, channel, monCtx);
- return;
- }
-
- // If is a retry, we must check if it is a already committed request abort.
- // This can happen because a client could have missed the reply, so it
- // retried the request after a timeout. So we added to the batch and when
- // it's flushed we'll add events to the retry processor in order to check
- // for false positive aborts. It needs to be done after the flush in case
- // the commit has occurred but it hasn't been persisted yet.
- batch.addUndecidedRetriedRequest(startTimestamp, channel, monCtx);
- }
-
- // no event has been received in the timeout period
- @Override
- public void onTimeout(final long sequence) {
- maybeFlushBatch();
- }
-
- /**
- * Flush the current batch if it's full, or the timeout has been elapsed since the last flush.
- */
- private void maybeFlushBatch() {
- if (batch.isFull()) {
- flush();
- } else if ((System.nanoTime() - lastFlush) > TimeUnit.MILLISECONDS.toNanos(batchPersistTimeoutInMs)) {
- timeoutMeter.mark();
- flush();
- }
- }
-
- @VisibleForTesting
- synchronized void flush() {
- lastFlush = System.nanoTime();
-
- boolean areWeStillMaster = true;
- if (!leaseManager.stillInLeasePeriod()) {
- // The master TSO replica has changed, so we must inform the
- // clients about it when sending the replies and avoid flushing
- // the current batch of TXs
- areWeStillMaster = false;
- // We need also to clear the data in the buffer
- writer.clearWriteBuffer();
- LOG.trace("Replica {} lost mastership before flushing data", tsoHostAndPort);
- } else {
- try {
- writer.flush();
- } catch (IOException e) {
- panicker.panic("Error persisting commit batch", e.getCause());
- }
- batchSizeHistogram.update(batch.getNumEvents());
- if (!leaseManager.stillInLeasePeriod()) {
- // If after flushing this TSO server is not the master
- // replica we need inform the client about it
- areWeStillMaster = false;
- LOG.warn("Replica {} lost mastership after flushing data", tsoHostAndPort);
- }
- }
- flushTimer.update((System.nanoTime() - lastFlush));
- batch.sendRepliesAndReset(reply, retryProc, areWeStillMaster);
-
- }
-
- @Override
- public void persistCommit(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) {
- long seq = persistRing.next();
- PersistEvent e = persistRing.get(seq);
- PersistEvent.makePersistCommit(e, startTimestamp, commitTimestamp, c, monCtx);
- persistRing.publish(seq);
- }
-
- @Override
- public void persistAbort(long startTimestamp, boolean isRetry, Channel c, MonitoringContext monCtx) {
- long seq = persistRing.next();
- PersistEvent e = persistRing.get(seq);
- PersistEvent.makePersistAbort(e, startTimestamp, isRetry, c, monCtx);
- persistRing.publish(seq);
- }
-
- @Override
- public void persistTimestamp(long startTimestamp, Channel c, MonitoringContext monCtx) {
- long seq = persistRing.next();
- PersistEvent e = persistRing.get(seq);
- PersistEvent.makePersistTimestamp(e, startTimestamp, c, monCtx);
- persistRing.publish(seq);
- }
-
- @Override
- public void persistLowWatermark(long lowWatermark) {
- try {
- writer.updateLowWatermark(lowWatermark);
- } catch (IOException e) {
- LOG.error("Should not be thrown");
- }
- }
-
- public static class Batch {
-
- final PersistEvent[] events;
- final int maxBatchSize;
- int numEvents;
-
- Batch(int maxBatchSize) {
- assert (maxBatchSize > 0);
- this.maxBatchSize = maxBatchSize;
- events = new PersistEvent[maxBatchSize];
- numEvents = 0;
- for (int i = 0; i < maxBatchSize; i++) {
- events[i] = new PersistEvent();
- }
- }
-
- boolean isFull() {
- assert (numEvents <= maxBatchSize);
- return numEvents == maxBatchSize;
- }
-
- int getNumEvents() {
- return numEvents;
- }
-
- void addCommit(long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) {
- if (isFull()) {
- throw new IllegalStateException("batch full");
- }
- int index = numEvents++;
- PersistEvent e = events[index];
- PersistEvent.makePersistCommit(e, startTimestamp, commitTimestamp, c, monCtx);
- }
-
- void addUndecidedRetriedRequest(long startTimestamp, Channel c, MonitoringContext monCtx) {
- if (isFull()) {
- throw new IllegalStateException("batch full");
- }
- int index = numEvents++;
- PersistEvent e = events[index];
- // We mark the event as an ABORT retry to identify the events to send
- // to the retry processor
- PersistEvent.makePersistAbort(e, startTimestamp, true, c, monCtx);
- }
-
- void addTimestamp(long startTimestamp, Channel c, MonitoringContext monCtx) {
- if (isFull()) {
- throw new IllegalStateException("batch full");
- }
- int index = numEvents++;
- PersistEvent e = events[index];
- PersistEvent.makePersistTimestamp(e, startTimestamp, c, monCtx);
- }
-
- void sendRepliesAndReset(ReplyProcessor reply, RetryProcessor retryProc, boolean isTSOInstanceMaster) {
- for (int i = 0; i < numEvents; i++) {
- PersistEvent e = events[i];
- switch (e.getType()) {
- case TIMESTAMP:
- e.getMonCtx().timerStop("timestampPersistProcessor");
- reply.timestampResponse(e.getStartTimestamp(), e.getChannel(), e.getMonCtx());
- break;
- case COMMIT:
- e.getMonCtx().timerStop("commitPersistProcessor");
- if (isTSOInstanceMaster) {
- reply.commitResponse(false, e.getStartTimestamp(), e.getCommitTimestamp(), e.getChannel(),
- e.getMonCtx());
- } else {
- // The client will need to perform heuristic actions to determine the output
- reply.commitResponse(true, e.getStartTimestamp(), e.getCommitTimestamp(), e.getChannel(),
- e.getMonCtx());
- }
- break;
- case ABORT:
- if (e.isRetry()) {
- retryProc.disambiguateRetryRequestHeuristically(e.getStartTimestamp(), e.getChannel(),
- e.getMonCtx());
- } else {
- LOG.error("We should not be receiving non-retried aborted requests in here");
- }
- break;
- default:
- LOG.error("We should receive only COMMIT or ABORT event types. Received {}", e.getType());
- break;
- }
- }
- numEvents = 0;
- }
-
- }
-
- public final static class PersistEvent {
-
- private MonitoringContext monCtx;
-
- enum Type {
- TIMESTAMP, COMMIT, ABORT
- }
-
- private Type type = null;
- private Channel channel = null;
-
- private boolean isRetry = false;
- private long startTimestamp = 0;
- private long commitTimestamp = 0;
-
- static void makePersistCommit(PersistEvent e, long startTimestamp, long commitTimestamp, Channel c,
- MonitoringContext monCtx) {
- e.type = Type.COMMIT;
- e.startTimestamp = startTimestamp;
- e.commitTimestamp = commitTimestamp;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- static void makePersistAbort(PersistEvent e, long startTimestamp, boolean isRetry, Channel c,
- MonitoringContext monCtx) {
- e.type = Type.ABORT;
- e.startTimestamp = startTimestamp;
- e.isRetry = isRetry;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- static void makePersistTimestamp(PersistEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
- e.type = Type.TIMESTAMP;
- e.startTimestamp = startTimestamp;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- MonitoringContext getMonCtx() {
- return monCtx;
- }
-
- Type getType() {
- return type;
- }
-
- Channel getChannel() {
- return channel;
- }
-
- boolean isRetry() {
- return isRetry;
- }
-
- long getStartTimestamp() {
- return startTimestamp;
- }
-
- long getCommitTimestamp() {
- return commitTimestamp;
- }
-
- public final static EventFactory<PersistEvent> EVENT_FACTORY = new EventFactory<PersistEvent>() {
- @Override
- public PersistEvent newInstance() {
- return new PersistEvent();
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
deleted file mode 100644
index f0788ed..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.jboss.netty.channel.Channel;
-
-interface ReplyProcessor {
- /**
- * Informs the client about the outcome of the Tx it was trying to
- * commit. If the heuristic decision flat is enabled, the client
- * will need to do additional actions for learning the final outcome.
- *
- * @param makeHeuristicDecision
- * informs about whether heuristic actions are needed or not
- * @param startTimestamp
- * the start timestamp of the transaction (a.k.a. tx id)
- * @param commitTimestamp
- * the commit timestamp of the transaction
- * @param channel
- * the communication channed with the client
- */
- void commitResponse(boolean makeHeuristicDecision, long startTimestamp, long commitTimestamp, Channel channel, MonitoringContext monCtx);
-
- void abortResponse(long startTimestamp, Channel c, MonitoringContext monCtx);
-
- void timestampResponse(long startTimestamp, Channel c, MonitoringContext monCtx);
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
deleted file mode 100644
index a69700d..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/ReplyProcessorImpl.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.lmax.disruptor.BatchEventProcessor;
-import com.lmax.disruptor.BusySpinWaitStrategy;
-import com.lmax.disruptor.EventFactory;
-import com.lmax.disruptor.EventHandler;
-import com.lmax.disruptor.RingBuffer;
-import com.lmax.disruptor.SequenceBarrier;
-import com.yahoo.omid.metrics.Meter;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.proto.TSOProto;
-import org.jboss.netty.channel.Channel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-
-class ReplyProcessorImpl implements EventHandler<ReplyProcessorImpl.ReplyEvent>, ReplyProcessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(ReplyProcessorImpl.class);
-
- final RingBuffer<ReplyEvent> replyRing;
- final Meter abortMeter;
- final Meter commitMeter;
- final Meter timestampMeter;
-
- @Inject
- ReplyProcessorImpl(MetricsRegistry metrics, Panicker panicker) {
- replyRing = RingBuffer.<ReplyEvent>createMultiProducer(ReplyEvent.EVENT_FACTORY, 1 << 12,
- new BusySpinWaitStrategy());
- SequenceBarrier replySequenceBarrier = replyRing.newBarrier();
- BatchEventProcessor<ReplyEvent> replyProcessor = new BatchEventProcessor<ReplyEvent>(
- replyRing, replySequenceBarrier, this);
- replyProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
-
- replyRing.addGatingSequences(replyProcessor.getSequence());
-
- ExecutorService replyExec = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("reply-%d").build());
- replyExec.submit(replyProcessor);
-
- abortMeter = metrics.meter(name("tso", "aborts"));
- commitMeter = metrics.meter(name("tso", "commits"));
- timestampMeter = metrics.meter(name("tso", "timestampAllocation"));
- }
-
- public void onEvent(ReplyEvent event, long sequence, boolean endOfBatch) throws Exception {
- String name = null;
- try {
- switch (event.getType()) {
- case COMMIT:
- name = "commitReplyProcessor";
- event.getMonCtx().timerStart(name);
- handleCommitResponse(false, event.getStartTimestamp(), event.getCommitTimestamp(), event.getChannel());
- break;
- case HEURISTIC_COMMIT:
- name = "commitReplyProcessor";
- event.getMonCtx().timerStart(name);
- handleCommitResponse(true, event.getStartTimestamp(), event.getCommitTimestamp(), event.getChannel());
- break;
- case ABORT:
- name = "abortReplyProcessor";
- event.getMonCtx().timerStart(name);
- handleAbortResponse(event.getStartTimestamp(), event.getChannel());
- break;
- case TIMESTAMP:
- name = "timestampReplyProcessor";
- event.getMonCtx().timerStart(name);
- handleTimestampResponse(event.getStartTimestamp(), event.getChannel());
- break;
- default:
- LOG.error("Unknown event {}", event.getType());
- break;
- }
- } finally {
- if (name != null) {
- event.getMonCtx().timerStop(name);
- }
- }
- event.getMonCtx().publish();
- }
-
- @Override
- public void commitResponse(boolean makeHeuristicDecision, long startTimestamp, long commitTimestamp, Channel c, MonitoringContext monCtx) {
- long seq = replyRing.next();
- ReplyEvent e = replyRing.get(seq);
- ReplyEvent.makeCommitResponse(makeHeuristicDecision, e, startTimestamp, commitTimestamp, c, monCtx);
- replyRing.publish(seq);
- }
-
- @Override
- public void abortResponse(long startTimestamp, Channel c, MonitoringContext monCtx) {
- long seq = replyRing.next();
- ReplyEvent e = replyRing.get(seq);
- ReplyEvent.makeAbortResponse(e, startTimestamp, c, monCtx);
- replyRing.publish(seq);
- }
-
- @Override
- public void timestampResponse(long startTimestamp, Channel c, MonitoringContext monCtx) {
- long seq = replyRing.next();
- ReplyEvent e = replyRing.get(seq);
- ReplyEvent.makeTimestampReponse(e, startTimestamp, c, monCtx);
- replyRing.publish(seq);
- }
-
- void handleCommitResponse(boolean makeHeuristicDecision, long startTimestamp, long commitTimestamp, Channel c) {
- TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
- TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
- if (makeHeuristicDecision) { // If the commit is ambiguous is due to a new master TSO
- commitBuilder.setMakeHeuristicDecision(true);
- }
- commitBuilder.setAborted(false)
- .setStartTimestamp(startTimestamp)
- .setCommitTimestamp(commitTimestamp);
- builder.setCommitResponse(commitBuilder.build());
- c.write(builder.build());
-
- commitMeter.mark();
- }
-
- void handleAbortResponse(long startTimestamp, Channel c) {
- TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
- TSOProto.CommitResponse.Builder commitBuilder = TSOProto.CommitResponse.newBuilder();
- commitBuilder.setAborted(true)
- .setStartTimestamp(startTimestamp);
- builder.setCommitResponse(commitBuilder.build());
- c.write(builder.build());
-
- abortMeter.mark();
- }
-
- void handleTimestampResponse(long startTimestamp, Channel c) {
- TSOProto.Response.Builder builder = TSOProto.Response.newBuilder();
- TSOProto.TimestampResponse.Builder respBuilder = TSOProto.TimestampResponse.newBuilder();
- respBuilder.setStartTimestamp(startTimestamp);
- builder.setTimestampResponse(respBuilder.build());
- c.write(builder.build());
-
- timestampMeter.mark();
- }
-
- public final static class ReplyEvent {
-
- enum Type {
- TIMESTAMP, COMMIT, HEURISTIC_COMMIT, ABORT
- }
-
- private Type type = null;
- private Channel channel = null;
-
- private long startTimestamp = 0;
- private long commitTimestamp = 0;
- private MonitoringContext monCtx;
-
- Type getType() {
- return type;
- }
-
- Channel getChannel() {
- return channel;
- }
-
- long getStartTimestamp() {
- return startTimestamp;
- }
-
- long getCommitTimestamp() {
- return commitTimestamp;
- }
-
- MonitoringContext getMonCtx() {
- return monCtx;
- }
-
- static void makeTimestampReponse(ReplyEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
- e.type = Type.TIMESTAMP;
- e.startTimestamp = startTimestamp;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- static void makeCommitResponse(boolean makeHeuristicDecision, ReplyEvent e, long startTimestamp,
- long commitTimestamp, Channel c, MonitoringContext monCtx) {
-
- if (makeHeuristicDecision) {
- e.type = Type.HEURISTIC_COMMIT;
- } else {
- e.type = Type.COMMIT;
- }
- e.startTimestamp = startTimestamp;
- e.commitTimestamp = commitTimestamp;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- static void makeAbortResponse(ReplyEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
- e.type = Type.ABORT;
- e.startTimestamp = startTimestamp;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- public final static EventFactory<ReplyEvent> EVENT_FACTORY = new EventFactory<ReplyEvent>() {
- @Override
- public ReplyEvent newInstance() {
- return new ReplyEvent();
- }
- };
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
deleted file mode 100644
index 2870cb7..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessor.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.jboss.netty.channel.Channel;
-
-import java.util.Collection;
-
-// NOTE: public is required explicitly in the interface definition for Guice injection
-public interface RequestProcessor extends TSOStateManager.StateObserver {
-
- void timestampRequest(Channel c, MonitoringContext monCtx);
-
- void commitRequest(long startTimestamp, Collection<Long> writeSet, boolean isRetry, Channel c, MonitoringContext monCtx);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
deleted file mode 100644
index 6194b2e..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RequestProcessorImpl.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.lmax.disruptor.BatchEventProcessor;
-import com.lmax.disruptor.BusySpinWaitStrategy;
-import com.lmax.disruptor.EventFactory;
-import com.lmax.disruptor.EventHandler;
-import com.lmax.disruptor.RingBuffer;
-import com.lmax.disruptor.SequenceBarrier;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.tso.TSOStateManager.TSOState;
-import org.jboss.netty.channel.Channel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class RequestProcessorImpl implements EventHandler<RequestProcessorImpl.RequestEvent>, RequestProcessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(RequestProcessorImpl.class);
-
- private final TimestampOracle timestampOracle;
- public final CommitHashMap hashmap;
- private final MetricsRegistry metrics;
- private final PersistenceProcessor persistProc;
- private final RingBuffer<RequestEvent> requestRing;
- private long lowWatermark = -1L;
- private long epoch = -1L;
-
- @Inject
- RequestProcessorImpl(TSOServerConfig config,
- MetricsRegistry metrics,
- TimestampOracle timestampOracle,
- PersistenceProcessor persistProc,
- Panicker panicker) throws IOException {
-
- this.metrics = metrics;
-
- this.persistProc = persistProc;
- this.timestampOracle = timestampOracle;
-
- this.hashmap = new CommitHashMap(config.getMaxItems());
-
- // Set up the disruptor thread
- requestRing = RingBuffer.<RequestEvent>createMultiProducer(RequestEvent.EVENT_FACTORY, 1 << 12,
- new BusySpinWaitStrategy());
- SequenceBarrier requestSequenceBarrier = requestRing.newBarrier();
- BatchEventProcessor<RequestEvent> requestProcessor =
- new BatchEventProcessor<RequestEvent>(requestRing,
- requestSequenceBarrier,
- this);
- requestRing.addGatingSequences(requestProcessor.getSequence());
- requestProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
-
- ExecutorService requestExec = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("request-%d").build());
- // Each processor runs on a separate thread
- requestExec.submit(requestProcessor);
-
- }
-
- /**
- * This should be called when the TSO gets initialized or gets leadership
- */
- @Override
- public void update(TSOState state) throws IOException {
- LOG.info("Reseting RequestProcessor...");
- this.lowWatermark = state.getLowWatermark();
- persistProc.persistLowWatermark(lowWatermark);
- this.epoch = state.getEpoch();
- hashmap.reset();
- LOG.info("RequestProcessor initialized with LWMs {} and Epoch {}", lowWatermark, epoch);
- }
-
- @Override
- public void onEvent(RequestEvent event, long sequence, boolean endOfBatch) throws Exception {
- String name = null;
- try {
- if (event.getType() == RequestEvent.Type.TIMESTAMP) {
- name = "timestampReqProcessor";
- event.getMonCtx().timerStart(name);
- handleTimestamp(event);
- } else if (event.getType() == RequestEvent.Type.COMMIT) {
- name = "commitReqProcessor";
- event.getMonCtx().timerStart(name);
- handleCommit(event);
- }
- } finally {
- if (name != null) {
- event.getMonCtx().timerStop(name);
- }
- }
-
- }
-
- @Override
- public void timestampRequest(Channel c, MonitoringContext monCtx) {
- long seq = requestRing.next();
- RequestEvent e = requestRing.get(seq);
- RequestEvent.makeTimestampRequest(e, c, monCtx);
- requestRing.publish(seq);
- }
-
- @Override
- public void commitRequest(long startTimestamp, Collection<Long> writeSet, boolean isRetry, Channel c, MonitoringContext monCtx) {
- long seq = requestRing.next();
- RequestEvent e = requestRing.get(seq);
- RequestEvent.makeCommitRequest(e, startTimestamp, monCtx, writeSet, isRetry, c);
- requestRing.publish(seq);
- }
-
- public void handleTimestamp(RequestEvent requestEvent) {
- long timestamp;
-
- try {
- timestamp = timestampOracle.next();
- } catch (IOException e) {
- LOG.error("Error getting timestamp", e);
- return;
- }
-
- persistProc.persistTimestamp(timestamp, requestEvent.getChannel(), requestEvent.getMonCtx());
- }
-
- public long handleCommit(RequestEvent event) {
- long startTimestamp = event.getStartTimestamp();
- Iterable<Long> writeSet = event.writeSet();
- boolean isRetry = event.isRetry();
- Channel c = event.getChannel();
-
- boolean committed = false;
- long commitTimestamp = 0L;
-
- int numCellsInWriteset = 0;
- // 0. check if it should abort
- if (startTimestamp <= lowWatermark) {
- committed = false;
- } else {
- // 1. check the write-write conflicts
- committed = true;
- for (long cellId : writeSet) {
- long value = hashmap.getLatestWriteForCell(cellId);
- if (value != 0 && value >= startTimestamp) {
- committed = false;
- break;
- }
- numCellsInWriteset++;
- }
- }
-
- if (committed) {
- // 2. commit
- try {
- commitTimestamp = timestampOracle.next();
-
- if (numCellsInWriteset > 0) {
- long newLowWatermark = lowWatermark;
-
- for (long r : writeSet) {
- long removed = hashmap.putLatestWriteForCell(r, commitTimestamp);
- newLowWatermark = Math.max(removed, newLowWatermark);
- }
-
- if (newLowWatermark != lowWatermark) {
- LOG.trace("Setting new low Watermark to {}", newLowWatermark);
- lowWatermark = newLowWatermark;
- persistProc.persistLowWatermark(newLowWatermark);
- }
- }
- persistProc.persistCommit(startTimestamp, commitTimestamp, c, event.getMonCtx());
- } catch (IOException e) {
- LOG.error("Error committing", e);
- }
- } else { // add it to the aborted list
- persistProc.persistAbort(startTimestamp, isRetry, c, event.getMonCtx());
- }
-
- return commitTimestamp;
- }
-
- final static class RequestEvent implements Iterable<Long> {
-
- enum Type {
- TIMESTAMP, COMMIT
- }
-
- ;
-
- private Type type = null;
- private Channel channel = null;
-
- private boolean isRetry = false;
- private long startTimestamp = 0;
- private MonitoringContext monCtx;
- private long numCells = 0;
-
- private static final int MAX_INLINE = 40;
- private Long writeSet[] = new Long[MAX_INLINE];
- private Collection<Long> writeSetAsCollection = null; // for the case where there's more than MAX_INLINE
-
- static void makeTimestampRequest(RequestEvent e, Channel c, MonitoringContext monCtx) {
- e.type = Type.TIMESTAMP;
- e.channel = c;
- e.monCtx = monCtx;
- }
-
- static void makeCommitRequest(RequestEvent e,
- long startTimestamp, MonitoringContext monCtx, Collection<Long> writeSet,
- boolean isRetry, Channel c) {
- e.monCtx = monCtx;
- e.type = Type.COMMIT;
- e.channel = c;
- e.startTimestamp = startTimestamp;
- e.isRetry = isRetry;
- if (writeSet.size() > MAX_INLINE) {
- e.numCells = writeSet.size();
- e.writeSetAsCollection = writeSet;
- } else {
- e.writeSetAsCollection = null;
- e.numCells = writeSet.size();
- int i = 0;
- for (Long cellId : writeSet) {
- e.writeSet[i] = cellId;
- i++;
- }
- }
- }
-
- MonitoringContext getMonCtx() {
- return monCtx;
- }
-
- Type getType() {
- return type;
- }
-
- long getStartTimestamp() {
- return startTimestamp;
- }
-
- Channel getChannel() {
- return channel;
- }
-
- @Override
- public Iterator<Long> iterator() {
- if (writeSetAsCollection != null) {
- return writeSetAsCollection.iterator();
- }
- return new Iterator<Long>() {
- int i = 0;
-
- @Override
- public boolean hasNext() {
- return i < numCells;
- }
-
- @Override
- public Long next() {
- return writeSet[i++];
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
- Iterable<Long> writeSet() {
- return this;
- }
-
- boolean isRetry() {
- return isRetry;
- }
-
- public final static EventFactory<RequestEvent> EVENT_FACTORY
- = new EventFactory<RequestEvent>() {
- @Override
- public RequestEvent newInstance() {
- return new RequestEvent();
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java b/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
deleted file mode 100644
index 70309b9..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessor.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.jboss.netty.channel.Channel;
-
-interface RetryProcessor {
- void disambiguateRetryRequestHeuristically(long startTimestamp, Channel c, MonitoringContext monCtx);
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java b/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
deleted file mode 100644
index 13fbe2d..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/RetryProcessorImpl.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.lmax.disruptor.BatchEventProcessor;
-import com.lmax.disruptor.EventFactory;
-import com.lmax.disruptor.EventHandler;
-import com.lmax.disruptor.RingBuffer;
-import com.lmax.disruptor.SequenceBarrier;
-import com.lmax.disruptor.WaitStrategy;
-import com.lmax.disruptor.YieldingWaitStrategy;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.metrics.Meter;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import org.jboss.netty.channel.Channel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-
-/**
- * Manages the retry requests that clients can send when they did not received the response in the specified timeout
- */
-class RetryProcessorImpl implements EventHandler<RetryProcessorImpl.RetryEvent>, RetryProcessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(RetryProcessor.class);
-
- // Disruptor chain stuff
- final ReplyProcessor replyProc;
- final RingBuffer<RetryEvent> retryRing;
-
- final CommitTable.Client commitTableClient;
- final CommitTable.Writer writer;
-
- // Metrics
- final Meter retriesMeter;
-
- @Inject
- RetryProcessorImpl(MetricsRegistry metrics, CommitTable commitTable, ReplyProcessor replyProc, Panicker panicker)
- throws IOException
- {
-
- this.commitTableClient = commitTable.getClient();
- this.writer = commitTable.getWriter();
- this.replyProc = replyProc;
-
- WaitStrategy strategy = new YieldingWaitStrategy();
-
- retryRing = RingBuffer.createSingleProducer(RetryEvent.EVENT_FACTORY, 1 << 12, strategy);
- SequenceBarrier retrySeqBarrier = retryRing.newBarrier();
- BatchEventProcessor<RetryEvent> retryProcessor = new BatchEventProcessor<>(retryRing, retrySeqBarrier, this);
- retryProcessor.setExceptionHandler(new FatalExceptionHandler(panicker));
-
- retryRing.addGatingSequences(retryProcessor.getSequence());
-
- ExecutorService retryExec = Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("retry-%d").build());
- retryExec.submit(retryProcessor);
-
- // Metrics
- retriesMeter = metrics.meter(name("tso", "retries"));
- }
-
- @Override
- public void onEvent(final RetryEvent event, final long sequence, final boolean endOfBatch)
- throws Exception {
-
- switch (event.getType()) {
- case COMMIT:
- // TODO: What happens when the IOException is thrown?
- handleCommitRetry(event);
- break;
- default:
- assert (false);
- break;
- }
-
- }
-
- private void handleCommitRetry(RetryEvent event) throws InterruptedException, ExecutionException {
-
- long startTimestamp = event.getStartTimestamp();
-
- try {
- Optional<CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(startTimestamp).get();
- if (commitTimestamp.isPresent()) {
- if (commitTimestamp.get().isValid()) {
- LOG.trace("Valid commit TS found in Commit Table");
- replyProc.commitResponse(false, startTimestamp, commitTimestamp.get().getValue(),
- event.getChannel(), event.getMonCtx());
- } else {
- LOG.trace("Invalid commit TS found in Commit Table");
- replyProc.abortResponse(startTimestamp, event.getChannel(), event.getMonCtx());
- }
- } else {
- LOG.trace("No commit TS found in Commit Table");
- replyProc.abortResponse(startTimestamp, event.getChannel(), event.getMonCtx());
- }
- } catch (InterruptedException e) {
- LOG.error("Interrupted reading from commit table");
- Thread.currentThread().interrupt();
- } catch (ExecutionException e) {
- LOG.error("Error reading from commit table", e);
- }
-
- retriesMeter.mark();
- }
-
- @Override
- public void disambiguateRetryRequestHeuristically(long startTimestamp, Channel c, MonitoringContext monCtx) {
- long seq = retryRing.next();
- RetryEvent e = retryRing.get(seq);
- RetryEvent.makeCommitRetry(e, startTimestamp, c, monCtx);
- retryRing.publish(seq);
- }
-
- public final static class RetryEvent {
-
- enum Type {
- COMMIT
- }
-
- private Type type = null;
-
- private long startTimestamp = 0;
- private Channel channel = null;
- private MonitoringContext monCtx;
-
- static void makeCommitRetry(RetryEvent e, long startTimestamp, Channel c, MonitoringContext monCtx) {
- e.monCtx = monCtx;
- e.type = Type.COMMIT;
- e.startTimestamp = startTimestamp;
- e.channel = c;
- }
-
- MonitoringContext getMonCtx() {
- return monCtx;
- }
-
- Type getType() {
- return type;
- }
-
- Channel getChannel() {
- return channel;
- }
-
- long getStartTimestamp() {
- return startTimestamp;
- }
-
- public final static EventFactory<RetryEvent> EVENT_FACTORY
- = new EventFactory<RetryEvent>() {
- @Override
- public RetryEvent newInstance() {
- return new RetryEvent();
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java b/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
deleted file mode 100644
index 7cfd77a..0000000
--- a/tso-server/src/main/java/com/yahoo/omid/tso/SystemExitPanicker.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SystemExitPanicker implements Panicker {
- private static final Logger LOG = LoggerFactory.getLogger(SystemExitPanicker.class);
- private static final int PANIC_EXIT_CODE = 123;
-
- @Override
- public void panic(String reason, Throwable cause) {
- LOG.error("PANICKING: {}", reason, cause);
- System.exit(PANIC_EXIT_CODE);
- }
-}
[08/52] [abbrv] incubator-omid git commit: Rename tsoclient package
to tso.client
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
deleted file mode 100644
index d41517c..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientConnectionToTSO.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.tso.HALeaseManagementModule;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.VoidLeaseManagementModule;
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.test.TestingServer;
-import org.apache.curator.utils.CloseableUtils;
-import org.apache.zookeeper.KeeperException.NoNodeException;
-import org.apache.zookeeper.data.Stat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-
-import static com.yahoo.omid.tsoclient.OmidClientConfiguration.ConnType.HA;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-public class TestTSOClientConnectionToTSO {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientConnectionToTSO.class);
-
- // Constants and variables for component connectivity
- private static final String TSO_HOST = "localhost";
- private static final String CURRENT_TSO_PATH = "/current_tso_path";
- private static final String TSO_LEASE_PATH = "/tso_lease_path";
-
- private int tsoPortForTest;
- private String zkClusterForTest;
-
- private Injector injector = null;
-
- private TestingServer zkServer;
-
- private CuratorFramework zkClient;
- private TSOServer tsoServer;
-
- @BeforeMethod
- public void beforeMethod() throws Exception {
-
- tsoPortForTest = TestUtils.getFreeLocalPort();
-
- int zkPortForTest = TestUtils.getFreeLocalPort();
- zkClusterForTest = TSO_HOST + ":" + zkPortForTest;
- LOG.info("Starting ZK Server in port {}", zkPortForTest);
- zkServer = TestUtils.provideTestingZKServer(zkPortForTest);
- LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
-
- zkClient = TestUtils.provideConnectedZKClient(zkClusterForTest);
-
- Stat stat;
- try {
- zkClient.delete().forPath(CURRENT_TSO_PATH);
- stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
- assertNull(stat, CURRENT_TSO_PATH + " should not exist");
- } catch (NoNodeException e) {
- LOG.info("{} ZNode did not exist", CURRENT_TSO_PATH);
- }
-
- }
-
- @AfterMethod
- public void afterMethod() {
-
- zkClient.close();
-
- CloseableUtils.closeQuietly(zkServer);
- zkServer = null;
- LOG.info("ZK Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testUnsuccessfulConnectionToTSO() throws Exception {
-
- // When no HA node for TSOServer is found & no host:port config exists
- // we should get an exception when getting the client
- try {
- TSOClient.newInstance(new OmidClientConfiguration());
- } catch (IllegalArgumentException e) {
- // Expected
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testSuccessfulConnectionToTSOWithHostAndPort() throws Exception {
-
- // Launch a TSO WITHOUT publishing the address in HA...
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setMaxItems(1000);
- tsoConfig.setPort(tsoPortForTest);
- tsoConfig.setLeaseModule(new VoidLeaseManagementModule());
- injector = Guice.createInjector(new TSOMockModule(tsoConfig));
- LOG.info("Starting TSO");
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading TSO");
-
- // When no HA node for TSOServer is found we should get a connection
- // to the TSO through the host:port configured...
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString("localhost:" + tsoPortForTest);
- tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // ... so we should get responses from the methods
- Long startTS = tsoClient.getNewStartTimestamp().get();
- LOG.info("Start TS {} ", startTS);
- assertEquals(startTS.longValue(), 1);
-
- // Close the tsoClient connection and stop the TSO Server
- tsoClient.close().get();
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("TSO Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testSuccessfulConnectionToTSOThroughZK() throws Exception {
-
- // Launch a TSO publishing the address in HA...
- TSOServerConfig config = new TSOServerConfig();
- config.setMaxItems(1000);
- config.setPort(tsoPortForTest);
- config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
- injector = Guice.createInjector(new TSOMockModule(config));
- LOG.info("Starting TSO");
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading TSO");
-
- waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
-
- // When a HA node for TSOServer is found we should get a connection
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionType(HA);
- tsoClientConf.setConnectionString(zkClusterForTest);
- tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // ... so we should get responses from the methods
- Long startTS = tsoClient.getNewStartTimestamp().get();
- LOG.info("Start TS {} ", startTS);
- assertEquals(startTS.longValue(), 1);
-
- // Close the tsoClient connection and stop the TSO Server
- tsoClient.close().get();
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("TSO Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testSuccessOfTSOClientReconnectionsToARestartedTSOWithZKPublishing() throws Exception {
-
- // Start a TSO with HA...
- TSOServerConfig config = new TSOServerConfig();
- config.setMaxItems(1000);
- config.setPort(tsoPortForTest);
- config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
- injector = Guice.createInjector(new TSOMockModule(config));
- LOG.info("Starting Initial TSO");
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading TSO");
-
- waitTillTsoRegisters(injector.getInstance(CuratorFramework.class));
-
- // Then create the TSO Client under test...
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionType(HA);
- tsoClientConf.setConnectionString(zkClusterForTest);
- tsoClientConf.setZkCurrentTsoPath(CURRENT_TSO_PATH);
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // ... and check that initially we get responses from the methods
- Long startTS = tsoClient.getNewStartTimestamp().get();
- LOG.info("Start TS {} ", startTS);
- assertEquals(startTS.longValue(), 1);
-
- // Then stop the server...
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("Initial TSO Server Stopped");
-
- Thread.sleep(1500); // ...allow the client to receive disconnection event...
- // ... and check that we get a conn exception when trying to access the client
- try {
- startTS = tsoClient.getNewStartTimestamp().get();
- fail();
- } catch (ExecutionException e) {
- LOG.info("Exception expected");
- // Internal accessor to fsm to do the required checkings
- FsmImpl fsm = (FsmImpl) tsoClient.fsm;
- assertEquals(e.getCause().getClass(), ConnectionException.class);
- assertTrue(fsm.getState().getClass().equals(TSOClient.ConnectionFailedState.class)
- ||
- fsm.getState().getClass().equals(TSOClient.DisconnectedState.class));
- }
-
- // After that, simulate that a new TSO has been launched...
- Injector newInjector = Guice.createInjector(new TSOMockModule(config));
- LOG.info("Re-Starting again the TSO");
- tsoServer = newInjector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_HOST, tsoPortForTest, 100);
- LOG.info("Finished loading restarted TSO");
-
- // Finally re-check that, eventually, we can get a new value from the new TSO...
- boolean reconnectionActive = false;
- while (!reconnectionActive) {
- try {
- startTS = tsoClient.getNewStartTimestamp().get();
- reconnectionActive = true;
- } catch (ExecutionException e) {
- // Expected
- }
- }
- assertNotNull(startTS);
-
- // ...and stop the server
- tsoServer.stopAndWait();
- TestUtils.waitForSocketNotListening(TSO_HOST, tsoPortForTest, 1000);
- LOG.info("Restarted TSO Server Stopped");
- }
-
- private void waitTillTsoRegisters(CuratorFramework zkClient) throws Exception {
- while (true) {
- try {
- Stat stat = zkClient.checkExists().forPath(CURRENT_TSO_PATH);
- if (stat == null) {
- continue;
- }
- LOG.info("TSO registered in HA with path {}={}", CURRENT_TSO_PATH, stat.toString());
- if (stat.toString().length() == 0) {
- continue;
- }
- return;
- } catch (Exception e) {
- LOG.debug("TSO still has not registered yet, sleeping...", e);
- Thread.sleep(500);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
deleted file mode 100644
index 33b12a9..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientRequestAndResponseBehaviours.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.collect.Sets;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.tso.PausableTimestampOracle;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.TimestampOracle;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-public class TestTSOClientRequestAndResponseBehaviours {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTSOClientRequestAndResponseBehaviours.class);
-
- private static final String TSO_SERVER_HOST = "localhost";
- private static final int TSO_SERVER_PORT = 1234;
-
- private final static CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
- private final static CellId c2 = new DummyCellIdImpl(0xfeedcafeL);
-
- private final static Set<CellId> testWriteSet = Sets.newHashSet(c1, c2);
-
- private OmidClientConfiguration tsoClientConf;
-
- // Required infrastructure for TSOClient test
- private TSOServer tsoServer;
- private PausableTimestampOracle pausableTSOracle;
- private CommitTable commitTable;
-
- @BeforeClass
- public void setup() throws Exception {
-
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setMaxItems(1000);
- tsoConfig.setPort(TSO_SERVER_PORT);
- Module tsoServerMockModule = new TSOMockModule(tsoConfig);
- Injector injector = Guice.createInjector(tsoServerMockModule);
-
- LOG.info("==================================================================================================");
- LOG.info("======================================= Init TSO Server ==========================================");
- LOG.info("==================================================================================================");
-
- tsoServer = injector.getInstance(TSOServer.class);
- tsoServer.startAndWait();
- TestUtils.waitForSocketListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 100);
-
- LOG.info("==================================================================================================");
- LOG.info("===================================== TSO Server Initialized =====================================");
- LOG.info("==================================================================================================");
-
- pausableTSOracle = (PausableTimestampOracle) injector.getInstance(TimestampOracle.class);
- commitTable = injector.getInstance(CommitTable.class);
-
- }
-
- @AfterClass
- public void tearDown() throws Exception {
-
- tsoServer.stopAndWait();
- tsoServer = null;
- TestUtils.waitForSocketNotListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 1000);
-
- }
-
- @BeforeMethod
- public void beforeMethod() {
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
-
- this.tsoClientConf = tsoClientConf;
-
- }
-
- @AfterMethod
- public void afterMethod() {
-
- pausableTSOracle.resume();
-
- }
-
- /**
- * Test to ensure TSOClient timeouts are cancelled.
- * At some point a bug was detected because the TSOClient timeouts were not cancelled, and as timestamp requests
- * had no way to be correlated to timestamp responses, random requests were just timed out after a certain time.
- * We send a lot of timestamp requests, and wait for them to complete.
- * Ensure that the next request doesn't get hit by the timeouts of the previous
- * requests. (i.e. make sure we cancel timeouts)
- */
- @Test(timeOut = 30_000)
- public void testTimeoutsAreCancelled() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- int requestTimeoutInMs = 500;
- int requestMaxRetries = 5;
- LOG.info("Request timeout {} ms; Max retries {}", requestTimeoutInMs, requestMaxRetries);
- Future<Long> f = null;
- for (int i = 0; i < (requestMaxRetries * 10); i++) {
- f = client.getNewStartTimestamp();
- }
- if (f != null) {
- f.get();
- }
- pausableTSOracle.pause();
- long msToSleep = ((long) (requestTimeoutInMs * 0.75));
- LOG.info("Sleeping for {} ms", msToSleep);
- TimeUnit.MILLISECONDS.sleep(msToSleep);
- f = client.getNewStartTimestamp();
- msToSleep = ((long) (requestTimeoutInMs * 0.9));
- LOG.info("Sleeping for {} ms", msToSleep);
- TimeUnit.MILLISECONDS.sleep(msToSleep);
- LOG.info("Resuming");
- pausableTSOracle.resume();
- f.get();
-
- }
-
- @Test(timeOut = 30_000)
- public void testCommitGetsServiceUnavailableExceptionWhenCommunicationFails() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestMaxRetries(0);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- List<Long> startTimestamps = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- startTimestamps.add(client.getNewStartTimestamp().get());
- }
-
- pausableTSOracle.pause();
-
- List<Future<Long>> futures = new ArrayList<>();
- for (long s : startTimestamps) {
- futures.add(client.commit(s, Sets.<CellId>newHashSet()));
- }
- TSOClientAccessor.closeChannel(client);
-
- for (Future<Long> f : futures) {
- try {
- f.get();
- fail("Shouldn't be able to complete");
- } catch (ExecutionException ee) {
- assertTrue(ee.getCause() instanceof ServiceUnavailableException,
- "Should be a service unavailable exception");
- }
- }
- }
-
- /**
- * Test that if a client tries to make a request without handshaking, it will be disconnected.
- */
- @Test(timeOut = 30_000)
- public void testHandshakeBetweenOldClientAndCurrentServer() throws Exception {
-
- TSOClientRaw raw = new TSOClientRaw(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- TSOProto.Request request = TSOProto.Request.newBuilder()
- .setTimestampRequest(TSOProto.TimestampRequest.newBuilder().build())
- .build();
- raw.write(request);
- try {
- raw.getResponse().get();
- fail("Channel should be closed");
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), ConnectionException.class, "Should be channel closed exception");
- }
- raw.close();
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Test duplicate commits
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * This tests the case where messages arrive at the TSO out of order. This can happen in the case
- * the channel get dropped and the retry is done in a new channel. However, the TSO will respond with
- * aborted to the original message because the retry was already committed and it would be prohibitively
- * expensive to check all non-retry requests to see if they are already committed. For this reason
- * a client must ensure that if it is sending a retry due to a socket error, the previous channel
- * must be entirely closed so that it will not actually receive the abort response. TCP guarantees
- * that this doesn't happen in non-socket error cases.
- *
- */
- @Test(timeOut = 30_000)
- public void testOutOfOrderMessages() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long ts1 = client.getNewStartTimestamp().get();
-
- TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
- TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
- assertFalse(response1.getCommitResponse().getAborted(), "Retry Transaction should commit");
- assertTrue(response2.getCommitResponse().getAborted(), "Transaction should abort");
- }
-
- @Test(timeOut = 30_000)
- public void testDuplicateCommitAborting() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long ts1 = client.getNewStartTimestamp().get();
- long ts2 = client.getNewStartTimestamp().get();
- client.commit(ts2, testWriteSet).get();
-
- TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
- TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
- assertTrue(response1.getCommitResponse().getAborted(), "Transaction should abort");
- assertTrue(response2.getCommitResponse().getAborted(), "Retry commit should abort");
- }
-
- @Test(timeOut = 30_000)
- public void testDuplicateCommit() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long ts1 = client.getNewStartTimestamp().get();
-
- TSOProto.Response response1 = clientOneShot.makeRequest(createCommitRequest(ts1, false, testWriteSet));
- TSOProto.Response response2 = clientOneShot.makeRequest(createCommitRequest(ts1, true, testWriteSet));
- assertEquals(response2.getCommitResponse().getCommitTimestamp(),
- response1.getCommitResponse().getCommitTimestamp(),
- "Commit timestamp should be the same");
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Test TSOClient retry behaviour
- // ----------------------------------------------------------------------------------------------------------------
-
- @Test(timeOut = 30_000)
- public void testCommitCanSucceedWhenChannelDisconnected() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
-
- long ts1 = client.getNewStartTimestamp().get();
- pausableTSOracle.pause();
- TSOFuture<Long> future = client.commit(ts1, testWriteSet);
- TSOClientAccessor.closeChannel(client);
- pausableTSOracle.resume();
- future.get();
-
- }
-
- @Test(timeOut = 30_000)
- public void testCommitCanSucceedWithMultipleTimeouts() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestTimeoutInMs(100);
- testTSOClientConf.setRequestMaxRetries(10000);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- long ts1 = client.getNewStartTimestamp().get();
- pausableTSOracle.pause();
- TSOFuture<Long> future = client.commit(ts1, testWriteSet);
- TimeUnit.SECONDS.sleep(1);
- pausableTSOracle.resume();
- future.get();
- }
-
- @Test(timeOut = 30_000)
- public void testCommitFailWhenTSOIsDown() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestTimeoutInMs(100);
- testTSOClientConf.setRequestMaxRetries(10);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- long ts1 = client.getNewStartTimestamp().get();
- pausableTSOracle.pause();
- TSOFuture<Long> future = client.commit(ts1, testWriteSet);
- try {
- future.get();
- } catch (ExecutionException e) {
- assertEquals(e.getCause().getClass(), ServiceUnavailableException.class,
- "Should be a ServiceUnavailableExeption");
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testTimestampRequestSucceedWithMultipleTimeouts() throws Exception {
-
- OmidClientConfiguration testTSOClientConf = new OmidClientConfiguration();
- testTSOClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- testTSOClientConf.setRequestTimeoutInMs(100);
- testTSOClientConf.setRequestMaxRetries(10000);
- TSOClient client = TSOClient.newInstance(testTSOClientConf);
-
- pausableTSOracle.pause();
- Future<Long> future = client.getNewStartTimestamp();
- TimeUnit.SECONDS.sleep(1);
- pausableTSOracle.resume();
- future.get();
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // The next 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
- // (They exercise the communication protocol) TODO Remove???
- // ----------------------------------------------------------------------------------------------------------------
- @Test
- public void testCommitTimestampPresentInCommitTableReturnsCommit() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long tx1ST = client.getNewStartTimestamp().get();
-
- clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- assertFalse(response.getCommitResponse().getAborted(), "Transaction should be committed");
- assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
- assertEquals(response.getCommitResponse().getCommitTimestamp(), tx1ST + 1);
- }
-
- @Test
- public void testInvalidCommitTimestampPresentInCommitTableReturnsAbort() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long tx1ST = client.getNewStartTimestamp().get();
- // Invalidate the transaction
- commitTable.getClient().tryInvalidateTransaction(tx1ST);
-
- clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- assertTrue(response.getCommitResponse().getAborted(), "Transaction should be aborted");
- assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
- assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
- }
-
- @Test
- public void testCommitTimestampNotPresentInCommitTableReturnsAnAbort() throws Exception {
-
- TSOClient client = TSOClient.newInstance(tsoClientConf);
- TSOClientOneShot clientOneShot = new TSOClientOneShot(TSO_SERVER_HOST, TSO_SERVER_PORT);
-
- long tx1ST = client.getNewStartTimestamp().get();
-
- clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
-
- // Simulate remove entry from the commit table before exercise retry
- commitTable.getClient().completeTransaction(tx1ST);
-
- TSOProto.Response response = clientOneShot.makeRequest(createRetryCommitRequest(tx1ST));
- assertTrue(response.getCommitResponse().getAborted(), "Transaction should abort");
- assertFalse(response.getCommitResponse().getMakeHeuristicDecision());
- assertEquals(response.getCommitResponse().getCommitTimestamp(), 0);
- }
- // ----------------------------------------------------------------------------------------------------------------
- // The previous 3 tests are similar to the ones in TestRetryProcessor but checking the result on the TSOClient side
- // (They exercise the communication protocol) TODO Remove???
- // ----------------------------------------------------------------------------------------------------------------
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private TSOProto.Request createRetryCommitRequest(long ts) {
- return createCommitRequest(ts, true, testWriteSet);
- }
-
- private TSOProto.Request createCommitRequest(long ts, boolean retry, Set<CellId> writeSet) {
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
- commitBuilder.setStartTimestamp(ts);
- commitBuilder.setIsRetry(retry);
- for (CellId cell : writeSet) {
- commitBuilder.addCellId(cell.getCellId());
- }
- return builder.setCommitRequest(commitBuilder.build()).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
deleted file mode 100644
index 66c3628..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestTSOClientResponseHandling.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.yahoo.omid.tso.ProgrammableTSOServer;
-import com.yahoo.omid.tso.ProgrammableTSOServer.AbortResponse;
-import com.yahoo.omid.tso.ProgrammableTSOServer.CommitResponse;
-import com.yahoo.omid.tso.ProgrammableTSOServer.TimestampResponse;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.Assert.assertEquals;
-
-public class TestTSOClientResponseHandling {
-
- private static final int TSO_PORT = 4321;
- private static final long START_TS = 1L;
- private static final long COMMIT_TS = 2L;
-
- private ProgrammableTSOServer tsoServer = new ProgrammableTSOServer(TSO_PORT);
- // Client under test
- private TSOClient tsoClient;
-
- @BeforeClass
- public void configureAndCreateClient() throws IOException, InterruptedException {
-
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString("localhost:" + TSO_PORT);
- tsoClient = TSOClient.newInstance(tsoClientConf);
- }
-
- @BeforeMethod
- public void reset() {
- tsoServer.cleanResponses();
- }
-
- @Test
- public void testTimestampRequestReceivingASuccessfulResponse() throws Exception {
- // test request timestamp response returns a timestamp
-
- // Program the TSO to return an ad-hoc Timestamp response
- tsoServer.queueResponse(new TimestampResponse(START_TS));
-
- long startTS = tsoClient.getNewStartTimestamp().get();
- assertEquals(startTS, START_TS);
- }
-
- @Test
- public void testCommitRequestReceivingAnAbortResponse() throws Exception {
- // test commit request which is aborted on the server side
- // (e.g. due to conflicts with other transaction) throws an
- // execution exception with an AbortException as a cause
-
- // Program the TSO to return an Abort response
- tsoServer.queueResponse(new AbortResponse(START_TS));
-
- try {
- tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), AbortException.class);
- }
- }
-
- @Test
- public void testCommitRequestReceivingASuccessfulResponse() throws Exception {
- // test commit request which is successfully committed on the server
- // side returns a commit timestamp
-
- // Program the TSO to return an Commit response (with no required heuristic actions)
- tsoServer.queueResponse(new CommitResponse(false, START_TS, COMMIT_TS));
-
- long commitTS = tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
- assertEquals(commitTS, COMMIT_TS);
- }
-
- @Test
- public void testCommitRequestReceivingAHeuristicResponse() throws Exception {
- // test commit request which needs heuristic actions from the client
- // throws an execution exception with a NewTSOException as a cause
-
- // Program the TSO to return an Commit response requiring heuristic actions
- tsoServer.queueResponse(new CommitResponse(true, START_TS, COMMIT_TS));
- try {
- tsoClient.commit(START_TS, Collections.<CellId>emptySet()).get();
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), NewTSOException.class);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java b/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
deleted file mode 100644
index 641ab0a..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tsoclient/TestUnconnectedTSOClient.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import com.yahoo.omid.tsoclient.TSOClient.DisconnectedState;
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-import org.slf4j.Logger;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.collect.Sets.newHashSet;
-import static org.slf4j.LoggerFactory.getLogger;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-/**
- * Test the behavior of requests on a TSOClient component that is not connected to a TSO server.
- */
-public class TestUnconnectedTSOClient {
-
- private static final Logger LOG = getLogger(TestUnconnectedTSOClient.class);
-
- private static final int TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST = 2;
-
- @Test(timeOut = 30_000) // 30 secs
- public void testRequestsDoneOnAnUnconnectedTSOClientAlwaysReturn() throws Exception {
-
- OmidClientConfiguration tsoClientConf = new OmidClientConfiguration();
- tsoClientConf.setConnectionString("localhost:12345");
- tsoClientConf.setReconnectionDelayInSecs(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST);
-
- // Component under test
- TSOClient tsoClient = TSOClient.newInstance(tsoClientConf);
-
- // Internal accessor to fsm
- FsmImpl fsm = (FsmImpl) tsoClient.fsm;
-
- assertEquals(fsm.getState().getClass(), DisconnectedState.class);
-
- // Test requests to the 3 relevant methods in TSO client
-
- try {
- tsoClient.getNewStartTimestamp().get();
- fail();
- } catch (ExecutionException e) {
- LOG.info("Exception expected");
- assertEquals(e.getCause().getClass(), ConnectionException.class);
- TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
- assertEquals(fsm.getState().getClass(), DisconnectedState.class);
- }
-
- try {
- tsoClient.commit(1, newHashSet(new DummyCellIdImpl(0xdeadbeefL))).get();
- fail();
- } catch (ExecutionException e) {
- LOG.info("Exception expected");
- assertEquals(e.getCause().getClass(), ConnectionException.class);
- TimeUnit.SECONDS.sleep(TSO_RECONNECTION_DELAY_IN_SECS_FOR_TEST * 2);
- assertEquals(fsm.getState().getClass(), DisconnectedState.class);
- }
-
- tsoClient.close().get();
- LOG.info("No exception expected");
- }
-
-}
[16/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java b/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
new file mode 100644
index 0000000..34ced79
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestBatch.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.jboss.netty.channel.Channel;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.AssertJUnit;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+
+public class TestBatch {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestBatch.class);
+
+ private static final int BATCH_SIZE = 1000;
+ private MetricsRegistry metrics = new NullMetricsProvider();
+
+ @Mock
+ private Channel channel;
+ @Mock
+ private RetryProcessor retryProcessor;
+ @Mock
+ private ReplyProcessor replyProcessor;
+
+ // The batch element to test
+ private PersistenceProcessorImpl.Batch batch;
+
+ @BeforeMethod(alwaysRun = true, timeOut = 30_000)
+ public void initMocksAndComponents() {
+ MockitoAnnotations.initMocks(this);
+ batch = new PersistenceProcessorImpl.Batch(BATCH_SIZE);
+ }
+
+ @Test
+ public void testBatchFunctionality() {
+
+ // Required mocks
+ Channel channel = Mockito.mock(Channel.class);
+ ReplyProcessor replyProcessor = Mockito.mock(ReplyProcessor.class);
+ RetryProcessor retryProcessor = Mockito.mock(RetryProcessor.class);
+
+ // The batch element to test
+ PersistenceProcessorImpl.Batch batch = new PersistenceProcessorImpl.Batch(BATCH_SIZE);
+
+ // Test initial state is OK
+ AssertJUnit.assertFalse("Batch shouldn't be full", batch.isFull());
+ AssertJUnit.assertEquals("Num events should be 0", 0, batch.getNumEvents());
+
+ // Test adding a single commit event is OK
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+ monCtx.timerStart("commitPersistProcessor");
+ batch.addCommit(0, 1, channel, monCtx);
+ AssertJUnit.assertFalse("Batch shouldn't be full", batch.isFull());
+ AssertJUnit.assertEquals("Num events should be 1", 1, batch.getNumEvents());
+
+ // Test when filling the batch with events, batch is full
+ for (int i = 0; i < (BATCH_SIZE - 1); i++) {
+ if (i % 2 == 0) {
+ monCtx = new MonitoringContext(metrics);
+ monCtx.timerStart("timestampPersistProcessor");
+ batch.addTimestamp(i, channel, monCtx);
+ } else {
+ monCtx = new MonitoringContext(metrics);
+ monCtx.timerStart("commitPersistProcessor");
+ batch.addCommit(i, i + 1, channel, monCtx);
+ }
+ }
+ AssertJUnit.assertTrue("Batch should be full", batch.isFull());
+ AssertJUnit.assertEquals("Num events should be " + BATCH_SIZE, BATCH_SIZE, batch.getNumEvents());
+
+ // Test an exception is thrown when batch is full and a new element is going to be added
+ try {
+ monCtx = new MonitoringContext(metrics);
+ monCtx.timerStart("commitPersistProcessor");
+ batch.addCommit(0, 1, channel, new MonitoringContext(metrics));
+ Assert.fail("Should throw an IllegalStateException");
+ } catch (IllegalStateException e) {
+ AssertJUnit.assertEquals("message returned doesn't match", "batch full", e.getMessage());
+ LOG.debug("IllegalStateException catched properly");
+ }
+
+ // Test that sending replies empties the batch
+ final boolean MASTER_INSTANCE = true;
+ final boolean SHOULD_MAKE_HEURISTIC_DECISSION = true;
+ batch.sendRepliesAndReset(replyProcessor, retryProcessor, MASTER_INSTANCE);
+ verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
+ .timestampResponse(anyLong(), any(Channel.class), any(MonitoringContext.class));
+ verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
+ .commitResponse(eq(!SHOULD_MAKE_HEURISTIC_DECISSION), anyLong(), anyLong(),
+ any(Channel.class), any(MonitoringContext.class));
+ AssertJUnit.assertFalse("Batch shouldn't be full", batch.isFull());
+ AssertJUnit.assertEquals("Num events should be 0", 0, batch.getNumEvents());
+
+ }
+
+ @Test
+ public void testBatchFunctionalityWhenMastershipIsLost() {
+ Channel channel = Mockito.mock(Channel.class);
+
+ // Fill the batch with events till full
+ for (int i = 0; i < BATCH_SIZE; i++) {
+ if (i % 2 == 0) {
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+ monCtx.timerStart("timestampPersistProcessor");
+ batch.addTimestamp(i, channel, monCtx);
+ } else {
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+ monCtx.timerStart("commitPersistProcessor");
+ batch.addCommit(i, i + 1, channel, monCtx);
+ }
+ }
+
+ // Test that sending replies empties the batch also when the replica
+ // is NOT master and calls the ambiguousCommitResponse() method on the
+ // reply processor
+ final boolean MASTER_INSTANCE = true;
+ final boolean SHOULD_MAKE_HEURISTIC_DECISSION = true;
+ batch.sendRepliesAndReset(replyProcessor, retryProcessor, !MASTER_INSTANCE);
+ verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
+ .timestampResponse(anyLong(), any(Channel.class), any(MonitoringContext.class));
+ verify(replyProcessor, timeout(100).times(BATCH_SIZE / 2))
+ .commitResponse(eq(SHOULD_MAKE_HEURISTIC_DECISSION), anyLong(), anyLong(), any(Channel.class), any(
+ MonitoringContext.class));
+ assertFalse(batch.isFull(), "Batch shouldn't be full");
+ assertEquals(batch.getNumEvents(), 0, "Num events should be 0");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestLeaseManager.java b/tso-server/src/test/java/org/apache/omid/tso/TestLeaseManager.java
new file mode 100644
index 0000000..2eb4244
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestLeaseManager.java
@@ -0,0 +1,418 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.base.Charsets;
+import org.apache.omid.TestUtils;
+import org.apache.omid.tso.TSOStateManager.TSOState;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.utils.CloseableUtils;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.apache.omid.tso.client.TSOClient.DEFAULT_ZK_CLUSTER;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class TestLeaseManager {
+
+ private static final long DUMMY_EPOCH_1 = 1L;
+ private static final long DUMMY_EPOCH_2 = 2L;
+ private static final long DUMMY_EPOCH_3 = 3L;
+ private static final long DUMMY_LOW_WATERMARK_1 = DUMMY_EPOCH_1;
+ private static final long DUMMY_LOW_WATERMARK_2 = DUMMY_EPOCH_2;
+ private static final long DUMMY_LOW_WATERMARK_3 = DUMMY_EPOCH_3;
+
+ private static final String LEASE_MGR_ID_1 = "LM1";
+ private static final String LEASE_MGR_ID_2 = "LM2";
+ private static final String INSTANCE_ID_1 = "LM1" + "#";
+ private static final String INSTANCE_ID_2 = "LM2" + "#";
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestLeaseManager.class);
+
+ private static final long TEST_LEASE_PERIOD_IN_MS = 1000; // 1 second
+
+ private CuratorFramework zkClient;
+ private TestingServer zkServer;
+
+ @Mock
+ private Panicker panicker;
+
+ private PausableLeaseManager leaseManager1;
+ private PausableLeaseManager leaseManager2;
+
+ @BeforeClass
+ public void beforeClass() throws Exception {
+
+ LOG.info("Starting ZK Server");
+ zkServer = TestUtils.provideTestingZKServer();
+ LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
+
+ zkClient = TestUtils.provideConnectedZKClient(DEFAULT_ZK_CLUSTER);
+
+ }
+
+ @AfterClass
+ public void afterClass() throws Exception {
+
+ zkClient.close();
+
+ CloseableUtils.closeQuietly(zkServer);
+ zkServer = null;
+ LOG.info("ZK Server Stopped");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testErrorInitializingTSOStateExitsTheTSO() throws Exception {
+
+ final String TEST_TSO_LEASE_PATH = "/test0_tsolease";
+ final String TEST_CURRENT_TSO_PATH = "/test0_currenttso";
+
+ Panicker panicker = spy(new MockPanicker());
+
+ TSOChannelHandler tsoChannelHandler = mock(TSOChannelHandler.class);
+ TSOStateManager stateManager = mock(TSOStateManager.class);
+ when(stateManager.reset()).thenThrow(new IOException());
+ leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
+ tsoChannelHandler,
+ stateManager,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+ leaseManager1.startService();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ verify(panicker, timeout(2000).atLeastOnce()).panic(anyString(), any(IOException.class));
+
+ leaseManager1.stopService();
+
+ }
+
+ @Test(timeOut = 60000)
+ public void testLeaseHolderDoesNotChangeWhenPausedForALongTimeAndTheresNoOtherInstance()
+ throws Exception {
+
+ final String TEST_TSO_LEASE_PATH = "/test1_tsolease";
+ final String TEST_CURRENT_TSO_PATH = "/test1_currenttso";
+
+ // Launch the instance under test...
+ TSOChannelHandler tsoChannelHandler1 = mock(TSOChannelHandler.class);
+ TSOStateManager stateManager1 = mock(TSOStateManager.class);
+ when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
+ leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
+ tsoChannelHandler1,
+ stateManager1,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+ leaseManager1.startService();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... check is the lease holder
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
+ assertTrue(leaseManager1.stillInLeasePeriod());
+
+ // Then, pause instance when trying to renew lease...
+ leaseManager1.pausedInTryToRenewLeasePeriod();
+
+ // ...let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ...check that nothing changed...
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
+
+ // Finally, resume the instance...
+ leaseManager1.resume();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... and check again that nothing changed
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
+ assertTrue(leaseManager1.stillInLeasePeriod());
+
+ }
+
+ @Test(timeOut = 60_000)
+ public void testLeaseHolderDoesNotChangeWhenANewLeaseManagerIsUp() throws Exception {
+
+ final String TEST_TSO_LEASE_PATH = "/test2_tsolease";
+ final String TEST_CURRENT_TSO_PATH = "/test2_currenttso";
+
+ // Launch the master instance...
+ TSOChannelHandler tsoChannelHandler1 = mock(TSOChannelHandler.class);
+ TSOStateManager stateManager1 = mock(TSOStateManager.class);
+ when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
+ leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
+ tsoChannelHandler1,
+ stateManager1,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+
+ leaseManager1.startService();
+
+ // ...let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ...so it should be the current holder of the lease
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
+ assertTrue(leaseManager1.stillInLeasePeriod());
+
+ // Then launch another instance...
+ TSOChannelHandler tsoChannelHandler2 = mock(TSOChannelHandler.class);
+ TSOStateManager stateManager2 = mock(TSOStateManager.class);
+ when(stateManager2.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_2, DUMMY_EPOCH_2));
+ leaseManager2 = new PausableLeaseManager(LEASE_MGR_ID_2,
+ tsoChannelHandler2,
+ stateManager2,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+ leaseManager2.startService();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... and after the period, the first instance should be still the holder
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
+ assertTrue(leaseManager1.stillInLeasePeriod());
+ assertFalse(leaseManager2.stillInLeasePeriod());
+ }
+
+ @Test(timeOut = 60_000)
+ public void testLeaseHolderChangesWhenActiveLeaseManagerIsPaused() throws Exception {
+
+ final String TEST_TSO_LEASE_PATH = "/test3_tsolease";
+ final String TEST_CURRENT_TSO_PATH = "/test3_currenttso";
+
+ // Launch the master instance...
+ TSOChannelHandler tsoChannelHandler1 = mock(TSOChannelHandler.class);
+ TSOStateManager stateManager1 = mock(TSOStateManager.class);
+ when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
+ leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
+ tsoChannelHandler1,
+ stateManager1,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+
+ leaseManager1.startService();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... so it should be the current holder of the lease
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
+ assertTrue(leaseManager1.stillInLeasePeriod());
+
+ // Then launch another instance...
+ TSOChannelHandler tsoChannelHandler2 = mock(TSOChannelHandler.class);
+ TSOStateManager stateManager2 = mock(TSOStateManager.class);
+ when(stateManager2.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_2, DUMMY_EPOCH_2));
+ leaseManager2 = new PausableLeaseManager(LEASE_MGR_ID_2,
+ tsoChannelHandler2,
+ stateManager2,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+ leaseManager2.startService();
+
+ // ... and pause active lease manager...
+ leaseManager1.pausedInStillInLeasePeriod();
+
+ // ... and let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... and check that lease owner should have changed to the second instance
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_2);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_2 + "2");
+ assertTrue(leaseManager2.stillInLeasePeriod());
+
+ // Now, lets resume the first instance...
+ when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_3, DUMMY_EPOCH_3));
+ leaseManager1.resume();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // and check the lease owner is still the second instance (preserves the lease)
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_2);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_2 + "2");
+ assertFalse(leaseManager1.stillInLeasePeriod());
+ assertTrue(leaseManager2.stillInLeasePeriod());
+
+ // Finally, pause active lease manager when trying to renew lease...
+ leaseManager2.pausedInTryToRenewLeasePeriod();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... and check lease owner is has changed again to the first instance
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "3");
+ assertFalse(leaseManager2.stillInLeasePeriod());
+ assertTrue(leaseManager1.stillInLeasePeriod());
+
+ // Resume the second instance...
+ leaseManager2.resume();
+
+ // ... let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ // ... but the lease owner should still be the first instance
+ checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
+ checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "3");
+ assertFalse(leaseManager2.stillInLeasePeriod());
+ assertTrue(leaseManager1.stillInLeasePeriod());
+
+ }
+
+
+ @Test(timeOut = 40_000)
+ public void testLeaseManagerPanicsWhenUnexpectedInfoIsFoundInCurrentTSOZnode() throws Exception {
+
+ final String TEST_TSO_LEASE_PATH = "/test_wronginfo_tsolease";
+ final String TEST_CURRENT_TSO_PATH = "/test_wronginfo_currenttso";
+
+ Panicker panicker = spy(new MockPanicker());
+
+ // Launch the master instance...
+ TSOStateManager stateManager1 = mock(TSOStateManager.class);
+ when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
+ PausableLeaseManager leaseManager = new PausableLeaseManager(LEASE_MGR_ID_1,
+ mock(TSOChannelHandler.class),
+ stateManager1,
+ TEST_LEASE_PERIOD_IN_MS,
+ TEST_TSO_LEASE_PATH,
+ TEST_CURRENT_TSO_PATH,
+ zkClient,
+ panicker);
+
+ leaseManager.startService();
+ // ...and let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ leaseManager.pausedInTryToRenewLeasePeriod();
+
+ // 1st Panic test) Inject corrupted data in the ZNode, force reelection and test the panicker is exercised
+ zkClient.setData().forPath(TEST_CURRENT_TSO_PATH, "CorruptedData!!!".getBytes());
+
+ // ...and let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+ leaseManager.resume();
+ // ...and let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ ArgumentCaptor<IllegalArgumentException> trowableIAE = ArgumentCaptor.forClass(IllegalArgumentException.class);
+ verify(panicker).panic(anyString(), trowableIAE.capture());
+ assertTrue(trowableIAE.getValue() instanceof IllegalArgumentException);
+ assertTrue(trowableIAE.getValue().getMessage().contains("Incorrect TSO Info found"));
+
+ // 2nd Panic test) Simulate that a new master appeared in the meantime, force reelection
+ // and test the panicker is exercised
+ reset(panicker);
+ zkClient.setData().forPath(TEST_CURRENT_TSO_PATH, "newTSO:12345#10000".getBytes());
+
+ leaseManager.pausedInTryToRenewLeasePeriod();
+
+ // ...and let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+ leaseManager.resume();
+ // ...and let the test run for some time...
+ Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
+
+ ArgumentCaptor<LeaseManagement.LeaseManagementException> trowableLME =
+ ArgumentCaptor.forClass(LeaseManagement.LeaseManagementException.class);
+ verify(panicker).panic(anyString(), trowableLME.capture());
+ assertTrue(trowableLME.getValue() instanceof LeaseManagement.LeaseManagementException);
+ assertTrue(trowableLME.getValue().getMessage().contains("Another TSO replica was found"));
+ }
+
+ @Test(timeOut = 1000)
+ public void testNonHALeaseManager() throws Exception {
+
+ // Launch the instance...
+ VoidLeaseManager leaseManager = new VoidLeaseManager(mock(TSOChannelHandler.class),
+ mock(TSOStateManager.class));
+
+ leaseManager.startService();
+ assertTrue(leaseManager.stillInLeasePeriod());
+ leaseManager.stopService();
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Checkers
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private void checkLeaseHolder(String tsoLeasePath, String expectedLeaseHolder) throws Exception {
+ byte[] leaseHolderInBytes = zkClient.getData().forPath(tsoLeasePath);
+ String leaseHolder = new String(leaseHolderInBytes, Charsets.UTF_8);
+
+ assertEquals(leaseHolder, expectedLeaseHolder);
+ }
+
+ private void checkInstanceId(String currentTSOPath, String expectedInstanceId) throws Exception {
+ byte[] expectedInstanceIdInBytes = zkClient.getData().forPath(currentTSOPath);
+ String instanceId = new String(expectedInstanceIdInBytes, Charsets.UTF_8);
+
+ assertEquals(instanceId, expectedInstanceId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestLongCache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestLongCache.java b/tso-server/src/test/java/org/apache/omid/tso/TestLongCache.java
new file mode 100644
index 0000000..6a88686
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestLongCache.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.Test;
+
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class TestLongCache {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestLongCache.class);
+
+ private static final long TEST_VALUE = 1000;
+
+ private Random random = new Random(System.currentTimeMillis());
+
+ @Test
+ public void testAddAndGetElemsAndResetCache() {
+
+ // Cache configuration
+ final int CACHE_SIZE = 10_000_000;
+ final int CACHE_ASSOCIATIVITY = 32;
+ Cache cache = new LongCache(CACHE_SIZE, CACHE_ASSOCIATIVITY);
+
+ // After creation, cache values should be the default
+ for (int i = 0; i < 1000; i++) {
+ long position = random.nextLong();
+ assertEquals(cache.get(position), LongCache.RESET_VALUE);
+ }
+
+ Set<Long> testedKeys = new TreeSet<>();
+ // Populate some of the values
+ for (int i = 0; i < 1000; i++) {
+ long position = random.nextLong();
+ cache.set(position, TEST_VALUE);
+ testedKeys.add(position);
+ }
+
+ // Get the values and check them
+ for (long key : testedKeys) {
+ assertEquals(cache.get(key), TEST_VALUE);
+ }
+
+ // Reset cache and check the values are the default again
+ long startTimeInMs = System.currentTimeMillis();
+ cache.reset();
+ long endTimeInMs = System.currentTimeMillis();
+ long resetTimeInMs = endTimeInMs - startTimeInMs;
+ LOG.info("Time in reseting cache of {}/{} elems/asoc {}ms", CACHE_SIZE, CACHE_ASSOCIATIVITY, resetTimeInMs);
+
+ for (long key : testedKeys) {
+ assertEquals(cache.get(key), LongCache.RESET_VALUE);
+ }
+
+ }
+
+ @Test(timeOut = 10000)
+ public void testEntriesAge() {
+
+ final int entries = 1000;
+
+ Cache cache = new LongCache(entries, 16);
+
+ int removals = 0;
+ long totalAge = 0;
+ double tempStdDev = 0;
+ double tempAvg = 0;
+
+ int i = 0;
+ int largestDeletedTimestamp = 0;
+ for (; i < entries * 10; ++i) {
+ long removed = cache.set(random.nextLong(), i);
+ if (removed > largestDeletedTimestamp) {
+ largestDeletedTimestamp = (int) removed;
+ }
+ }
+
+ long time = System.nanoTime();
+ for (; i < entries * 100; ++i) {
+ long removed = cache.set(random.nextLong(), i);
+ if (removed > largestDeletedTimestamp) {
+ largestDeletedTimestamp = (int) removed;
+ }
+ int gap = i - ((int) largestDeletedTimestamp);
+ removals++;
+ totalAge += gap;
+ double oldAvg = tempAvg;
+ tempAvg += (gap - tempAvg) / removals;
+ tempStdDev += (gap - oldAvg) * (gap - tempAvg);
+ }
+ long elapsed = System.nanoTime() - time;
+ LOG.info("Elapsed (ms): " + (elapsed / (double) 1000));
+
+ double avgGap = totalAge / (double) removals;
+ LOG.info("Avg gap: " + (tempAvg));
+ LOG.info("Std dev gap: " + Math.sqrt((tempStdDev / entries)));
+ assertTrue("avgGap should be greater than entries * 0.6",
+ avgGap > entries * 0.6);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestPanicker.java b/tso-server/src/test/java/org/apache/omid/tso/TestPanicker.java
new file mode 100644
index 0000000..816194d
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestPanicker.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.timestamp.storage.TimestampStorage;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+
+@SuppressWarnings({"UnusedDeclaration"})
+public class TestPanicker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestPanicker.class);
+
+ @Mock
+ private CommitTable.Writer mockWriter;
+ @Mock
+ private MetricsRegistry metrics;
+
+ @BeforeMethod
+ public void initMocksAndComponents() {
+ MockitoAnnotations.initMocks(this);
+ }
+
+ @AfterMethod
+ void afterMethod() {
+ Mockito.reset(mockWriter);
+ }
+
+ // Note this test has been moved and refactored to TestTimestampOracle because
+ // it tests the behaviour of the TimestampOracle.
+ // Please, remove me in a future commit
+ @Test
+ public void testTimestampOraclePanic() throws Exception {
+ TimestampStorage storage = spy(new TimestampOracleImpl.InMemoryTimestampStorage());
+ Panicker panicker = spy(new MockPanicker());
+
+ doThrow(new RuntimeException("Out of memory")).when(storage).updateMaxTimestamp(anyLong(), anyLong());
+
+ final TimestampOracleImpl tso = new TimestampOracleImpl(metrics, storage, panicker);
+ tso.initialize();
+ Thread allocThread = new Thread("AllocThread") {
+ @Override
+ public void run() {
+ try {
+ while (true) {
+ tso.next();
+ }
+ } catch (IOException ioe) {
+ LOG.error("Shouldn't occur");
+ }
+ }
+ };
+ allocThread.start();
+
+ verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
+ }
+
+ // Note this test has been moved and refactored to TestPersistenceProcessor because
+ // it tests the behaviour of the PersistenceProcessor.
+ // Please, remove me in a future commit
+ @Test
+ public void testCommitTablePanic() throws Exception {
+ Panicker panicker = spy(new MockPanicker());
+
+ doThrow(new IOException("Unable to write@TestPanicker")).when(mockWriter).flush();
+
+ final CommitTable.Client mockClient = mock(CommitTable.Client.class);
+ CommitTable commitTable = new CommitTable() {
+ @Override
+ public Writer getWriter() {
+ return mockWriter;
+ }
+
+ @Override
+ public Client getClient() {
+ return mockClient;
+ }
+ };
+
+ LeaseManager leaseManager = mock(LeaseManager.class);
+ doReturn(true).when(leaseManager).stillInLeasePeriod();
+ PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
+ metrics,
+ "localhost:1234",
+ leaseManager,
+ commitTable,
+ mock(ReplyProcessor.class),
+ mock(RetryProcessor.class),
+ panicker);
+ proc.persistCommit(1, 2, null, new MonitoringContext(metrics));
+ verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
+ }
+
+ // Note this test has been moved and refactored to TestPersistenceProcessor because
+ // it tests the behaviour of the PersistenceProcessor.
+ // Please, remove me in a future commit
+ @Test
+ public void testRuntimeExceptionTakesDownDaemon() throws Exception {
+ Panicker panicker = spy(new MockPanicker());
+
+ final CommitTable.Writer mockWriter = mock(CommitTable.Writer.class);
+ doThrow(new RuntimeException("Kaboom!")).when(mockWriter).addCommittedTransaction(anyLong(), anyLong());
+
+ final CommitTable.Client mockClient = mock(CommitTable.Client.class);
+ CommitTable commitTable = new CommitTable() {
+ @Override
+ public Writer getWriter() {
+ return mockWriter;
+ }
+
+ @Override
+ public Client getClient() {
+ return mockClient;
+ }
+ };
+ PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
+ metrics,
+ "localhost:1234",
+ mock(LeaseManager.class),
+ commitTable,
+ mock(ReplyProcessor.class),
+ mock(RetryProcessor.class),
+ panicker);
+ proc.persistCommit(1, 2, null, new MonitoringContext(metrics));
+ verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java b/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
new file mode 100644
index 0000000..eb93277
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestPersistenceProcessor.java
@@ -0,0 +1,221 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.tso.PersistenceProcessorImpl.Batch;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+
+@SuppressWarnings({"UnusedDeclaration"})
+public class TestPersistenceProcessor {
+
+ @Mock
+ private Batch batch;
+ @Mock
+ private CommitTable.Writer mockWriter;
+ @Mock
+ private CommitTable.Client mockClient;
+ @Mock
+ private RetryProcessor retryProcessor;
+ @Mock
+ private ReplyProcessor replyProcessor;
+ @Mock
+ private Panicker panicker;
+
+ private MetricsRegistry metrics;
+ private CommitTable commitTable;
+
+ @BeforeMethod(alwaysRun = true, timeOut = 30_000)
+ public void initMocksAndComponents() throws Exception {
+
+ MockitoAnnotations.initMocks(this);
+
+ // Configure mock writer to flush successfully
+ doThrow(new IOException("Unable to write")).when(mockWriter).flush();
+
+ // Configure null metrics provider
+ metrics = new NullMetricsProvider();
+
+ // Configure commit table to return the mocked writer and client
+ commitTable = new CommitTable() {
+ @Override
+ public Writer getWriter() {
+ return mockWriter;
+ }
+
+ @Override
+ public Client getClient() {
+ return mockClient;
+ }
+ };
+ }
+
+ @AfterMethod
+ void afterMethod() {
+ Mockito.reset(mockWriter);
+ }
+
+ @Test
+ public void testCommitPersistenceWithNonHALeaseManager() throws Exception {
+
+ // Init a non-HA lease manager
+ VoidLeaseManager leaseManager = spy(new VoidLeaseManager(mock(TSOChannelHandler.class),
+ mock(TSOStateManager.class)));
+
+ TSOServerConfig tsoServerConfig = new TSOServerConfig();
+ tsoServerConfig.setBatchPersistTimeoutInMs(100);
+ // Component under test
+ PersistenceProcessor proc = new PersistenceProcessorImpl(tsoServerConfig,
+ metrics,
+ "localhost:1234",
+ batch,
+ leaseManager,
+ commitTable,
+ replyProcessor,
+ retryProcessor,
+ panicker);
+
+ // The non-ha lease manager always return true for
+ // stillInLeasePeriod(), so verify the batch sends replies as master
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+ proc.persistCommit(1, 2, null, monCtx);
+ verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
+ verify(batch, timeout(1000).times(2)).sendRepliesAndReset(any(ReplyProcessor.class),
+ any(RetryProcessor.class),
+ eq(true));
+ }
+
+ @Test
+ public void testCommitPersistenceWithHALeaseManager() throws Exception {
+
+ // Init a HA lease manager
+ LeaseManager leaseManager = mock(LeaseManager.class);
+
+ TSOServerConfig tsoServerConfig = new TSOServerConfig();
+ tsoServerConfig.setBatchPersistTimeoutInMs(100);
+ // Component under test
+ PersistenceProcessor proc = new PersistenceProcessorImpl(tsoServerConfig,
+ metrics,
+ "localhost:1234",
+ batch,
+ leaseManager,
+ commitTable,
+ replyProcessor,
+ retryProcessor,
+ panicker);
+
+ // Configure the lease manager to always return true for
+ // stillInLeasePeriod, so verify the batch sends replies as master
+ doReturn(true).when(leaseManager).stillInLeasePeriod();
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+ proc.persistCommit(1, 2, null, monCtx);
+ verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
+ verify(batch).sendRepliesAndReset(any(ReplyProcessor.class), any(RetryProcessor.class), eq(true));
+
+ // Configure the lease manager to always return true first and false
+ // later for stillInLeasePeriod, so verify the batch sends replies as
+ // non-master
+ reset(leaseManager);
+ reset(batch);
+ doReturn(true).doReturn(false).when(leaseManager).stillInLeasePeriod();
+ proc.persistCommit(1, 2, null, monCtx);
+ verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
+ verify(batch).sendRepliesAndReset(any(ReplyProcessor.class), any(RetryProcessor.class), eq(false));
+
+ // Configure the lease manager to always return false for
+ // stillInLeasePeriod, so verify the batch sends replies as non-master
+ reset(leaseManager);
+ reset(batch);
+ doReturn(false).when(leaseManager).stillInLeasePeriod();
+ proc.persistCommit(1, 2, null, monCtx);
+ verify(leaseManager, timeout(1000).times(1)).stillInLeasePeriod();
+ verify(batch).sendRepliesAndReset(any(ReplyProcessor.class), any(RetryProcessor.class), eq(false));
+
+ }
+
+ @Test
+ public void testCommitTableExceptionOnCommitPersistenceTakesDownDaemon() throws Exception {
+
+ // Init lease management (doesn't matter if HA or not)
+ LeaseManagement leaseManager = mock(LeaseManagement.class);
+ PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
+ metrics,
+ "localhost:1234",
+ leaseManager,
+ commitTable,
+ mock(ReplyProcessor.class),
+ mock(RetryProcessor.class),
+ panicker);
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+
+ // Configure lease manager to work normally
+ doReturn(true).when(leaseManager).stillInLeasePeriod();
+
+ // Configure commit table writer to explode when flushing changes to DB
+ doThrow(new IOException("Unable to write@TestPersistenceProcessor2")).when(mockWriter).flush();
+
+ // Check the panic is extended!
+ proc.persistCommit(1, 2, null, monCtx);
+ verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
+
+ }
+
+ @Test
+ public void testRuntimeExceptionOnCommitPersistenceTakesDownDaemon() throws Exception {
+
+ PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
+ metrics,
+ "localhost:1234",
+ mock(LeaseManagement.class),
+ commitTable,
+ replyProcessor,
+ retryProcessor,
+ panicker);
+
+ // Configure writer to explode with a runtime exception
+ doThrow(new RuntimeException("Kaboom!")).when(mockWriter).addCommittedTransaction(anyLong(), anyLong());
+ MonitoringContext monCtx = new MonitoringContext(metrics);
+
+ // Check the panic is extended!
+ proc.persistCommit(1, 2, null, monCtx);
+ verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestRequestProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestRequestProcessor.java b/tso-server/src/test/java/org/apache/omid/tso/TestRequestProcessor.java
new file mode 100644
index 0000000..c128beb
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestRequestProcessor.java
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.collect.Lists;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.jboss.netty.channel.Channel;
+import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class TestRequestProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRequestProcessor.class);
+
+ private static final int CONFLICT_MAP_SIZE = 1000;
+ private static final int CONFLICT_MAP_ASSOCIATIVITY = 32;
+
+ private MetricsRegistry metrics = new NullMetricsProvider();
+
+ private PersistenceProcessor persist;
+
+ private TSOStateManager stateManager;
+
+ // Request processor under test
+ private RequestProcessor requestProc;
+
+ @BeforeMethod
+ public void beforeMethod() throws Exception {
+
+ // Build the required scaffolding for the test
+ MetricsRegistry metrics = new NullMetricsProvider();
+
+ TimestampOracleImpl timestampOracle =
+ new TimestampOracleImpl(metrics, new TimestampOracleImpl.InMemoryTimestampStorage(), new MockPanicker());
+
+ stateManager = new TSOStateManagerImpl(timestampOracle);
+
+ persist = mock(PersistenceProcessor.class);
+
+ TSOServerConfig config = new TSOServerConfig();
+ config.setMaxItems(CONFLICT_MAP_SIZE);
+
+ requestProc = new RequestProcessorImpl(config, metrics, timestampOracle, persist, new MockPanicker());
+
+ // Initialize the state for the experiment
+ stateManager.register(requestProc);
+ stateManager.reset();
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTimestamp() throws Exception {
+
+ requestProc.timestampRequest(null, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> firstTScapture = ArgumentCaptor.forClass(Long.class);
+ verify(persist, timeout(100).times(1)).persistTimestamp(
+ firstTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+
+ long firstTS = firstTScapture.getValue();
+ // verify that timestamps increase monotonically
+ for (int i = 0; i < 100; i++) {
+ requestProc.timestampRequest(null, new MonitoringContext(metrics));
+ verify(persist, timeout(100).times(1)).persistTimestamp(eq(firstTS++), any(Channel.class), any(MonitoringContext.class));
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommit() throws Exception {
+
+ requestProc.timestampRequest(null, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> TScapture = ArgumentCaptor.forClass(Long.class);
+ verify(persist, timeout(100).times(1)).persistTimestamp(
+ TScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+ long firstTS = TScapture.getValue();
+
+ List<Long> writeSet = Lists.newArrayList(1L, 20L, 203L);
+ requestProc.commitRequest(firstTS - 1, writeSet, false, null, new MonitoringContext(metrics));
+ verify(persist, timeout(100).times(1)).persistAbort(eq(firstTS - 1), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
+
+ requestProc.commitRequest(firstTS, writeSet, false, null, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> commitTScapture = ArgumentCaptor.forClass(Long.class);
+
+ verify(persist, timeout(100).times(1)).persistCommit(eq(firstTS), commitTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+ assertTrue("Commit TS must be greater than start TS", commitTScapture.getValue() > firstTS);
+
+ // test conflict
+ requestProc.timestampRequest(null, new MonitoringContext(metrics));
+ TScapture = ArgumentCaptor.forClass(Long.class);
+ verify(persist, timeout(100).times(2)).persistTimestamp(
+ TScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+ long secondTS = TScapture.getValue();
+
+ requestProc.timestampRequest(null, new MonitoringContext(metrics));
+ TScapture = ArgumentCaptor.forClass(Long.class);
+ verify(persist, timeout(100).times(3)).persistTimestamp(
+ TScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+ long thirdTS = TScapture.getValue();
+
+ requestProc.commitRequest(thirdTS, writeSet, false, null, new MonitoringContext(metrics));
+ verify(persist, timeout(100).times(1)).persistCommit(eq(thirdTS), anyLong(), any(Channel.class), any(MonitoringContext.class));
+ requestProc.commitRequest(secondTS, writeSet, false, null, new MonitoringContext(metrics));
+ verify(persist, timeout(100).times(1)).persistAbort(eq(secondTS), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testCommitRequestAbortsWhenResettingRequestProcessorState() throws Exception {
+
+ List<Long> writeSet = Collections.emptyList();
+
+ // Start a transaction...
+ requestProc.timestampRequest(null, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> capturedTS = ArgumentCaptor.forClass(Long.class);
+ verify(persist, timeout(100).times(1)).persistTimestamp(capturedTS.capture(),
+ any(Channel.class),
+ any(MonitoringContext.class));
+ long startTS = capturedTS.getValue();
+
+ // ... simulate the reset of the RequestProcessor state (e.g. due to
+ // a change in mastership) and...
+ stateManager.reset();
+
+ // ...check that the transaction is aborted when trying to commit
+ requestProc.commitRequest(startTS, writeSet, false, null, new MonitoringContext(metrics));
+ verify(persist, timeout(100).times(1)).persistAbort(eq(startTS), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
+
+ }
+
+ @Test(timeOut = 5_000)
+ public void testLowWatermarkIsStoredOnlyWhenACacheElementIsEvicted() throws Exception {
+
+ final int ANY_START_TS = 1;
+ final long FIRST_COMMIT_TS_EVICTED = 1L;
+ final long NEXT_COMMIT_TS_THAT_SHOULD_BE_EVICTED = 2L;
+
+ // Fill the cache to provoke a cache eviction
+ for (long i = 0; i < CONFLICT_MAP_SIZE + CONFLICT_MAP_ASSOCIATIVITY; i++) {
+ long writeSetElementHash = i + 1; // This is to match the assigned CT: K/V in cache = WS Element Hash/CT
+ List<Long> writeSet = Lists.newArrayList(writeSetElementHash);
+ requestProc.commitRequest(ANY_START_TS, writeSet, false, null, new MonitoringContext(metrics));
+ }
+
+ Thread.currentThread().sleep(3000); // Allow the Request processor to finish the request processing
+
+ // Check that first time its called is on init
+ verify(persist, timeout(100).times(1)).persistLowWatermark(0L);
+ // Then, check it is called when cache is full and the first element is evicted (should be a 1)
+ verify(persist, timeout(100).times(1)).persistLowWatermark(FIRST_COMMIT_TS_EVICTED);
+ // Finally it should never be called with the next element
+ verify(persist, timeout(100).never()).persistLowWatermark(NEXT_COMMIT_TS_THAT_SHOULD_BE_EVICTED);
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestRetryProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestRetryProcessor.java b/tso-server/src/test/java/org/apache/omid/tso/TestRetryProcessor.java
new file mode 100644
index 0000000..9ba4f9d
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestRetryProcessor.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.base.Optional;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.committable.InMemoryCommitTable;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.jboss.netty.channel.Channel;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.testng.AssertJUnit.assertEquals;
+
+public class TestRetryProcessor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestRetryProcessor.class);
+
+ private MetricsRegistry metrics = new NullMetricsProvider();
+
+ private static long NON_EXISTING_ST_TX = 1000;
+ private static long ST_TX_1 = 0;
+ private static long CT_TX_1 = 1;
+ private static long ST_TX_2 = 2;
+
+ @Mock
+ private Channel channel;
+ @Mock
+ private ReplyProcessor replyProc;
+ @Mock
+ private Panicker panicker;
+
+ private CommitTable commitTable;
+
+ @BeforeMethod(alwaysRun = true, timeOut = 30_000)
+ public void initMocksAndComponents() {
+ MockitoAnnotations.initMocks(this);
+ // Init components
+ commitTable = new InMemoryCommitTable();
+ metrics = new NullMetricsProvider();
+
+ }
+
+ @Test(timeOut = 10_000)
+ public void testBasicFunctionality() throws Exception {
+
+ // The element to test
+ RetryProcessor retryProc = new RetryProcessorImpl(metrics, commitTable, replyProc, panicker);
+
+ // Test we'll reply with an abort for a retry request when the start timestamp IS NOT in the commit table
+ retryProc.disambiguateRetryRequestHeuristically(NON_EXISTING_ST_TX, channel, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> firstTScapture = ArgumentCaptor.forClass(Long.class);
+ verify(replyProc, timeout(100).times(1)).abortResponse(firstTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+
+ long startTS = firstTScapture.getValue();
+ assertEquals("Captured timestamp should be the same as NON_EXISTING_ST_TX", NON_EXISTING_ST_TX, startTS);
+
+ // Test we'll reply with a commit for a retry request when the start timestamp IS in the commit table
+ commitTable.getWriter().addCommittedTransaction(ST_TX_1, CT_TX_1); // Add a tx to commit table
+
+ retryProc.disambiguateRetryRequestHeuristically(ST_TX_1, channel, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> secondTScapture = ArgumentCaptor.forClass(Long.class);
+ verify(replyProc, timeout(100).times(1))
+ .commitResponse(eq(false), firstTScapture.capture(), secondTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+
+ startTS = firstTScapture.getValue();
+ long commitTS = secondTScapture.getValue();
+ assertEquals("Captured timestamp should be the same as ST_TX_1", ST_TX_1, startTS);
+ assertEquals("Captured timestamp should be the same as CT_TX_1", CT_TX_1, commitTS);
+ }
+
+ @Test(timeOut = 10_000)
+ public void testRetriedRequestForInvalidatedTransactionReturnsAnAbort() throws Exception {
+
+ // Invalidate the transaction
+ commitTable.getClient().tryInvalidateTransaction(ST_TX_1);
+
+ // Pre-start verification: Validate that the transaction is invalidated
+ // NOTE: This test should be in the a test class for InMemoryCommitTable
+ Optional<CommitTimestamp> invalidTxMarker = commitTable.getClient().getCommitTimestamp(ST_TX_1).get();
+ Assert.assertTrue(invalidTxMarker.isPresent());
+ Assert.assertEquals(invalidTxMarker.get().getValue(), InMemoryCommitTable.INVALID_TRANSACTION_MARKER);
+
+ // The element to test
+ RetryProcessor retryProc = new RetryProcessorImpl(metrics, commitTable, replyProc, panicker);
+
+ // Test we'll reply with an abort for a retry request when the
+ // transaction id IS in the commit table BUT invalidated
+ retryProc.disambiguateRetryRequestHeuristically(ST_TX_1, channel, new MonitoringContext(metrics));
+ ArgumentCaptor<Long> startTScapture = ArgumentCaptor.forClass(Long.class);
+ verify(replyProc, timeout(100).times(1)).abortResponse(startTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
+
+ long startTS = startTScapture.getValue();
+ Assert.assertEquals(startTS, ST_TX_1, "Captured timestamp should be the same as NON_EXISTING_ST_TX");
+
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestTSOChannelHandlerNetty.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestTSOChannelHandlerNetty.java b/tso-server/src/test/java/org/apache/omid/tso/TestTSOChannelHandlerNetty.java
new file mode 100644
index 0000000..0f64181
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestTSOChannelHandlerNetty.java
@@ -0,0 +1,321 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.proto.TSOProto;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelException;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Matchers.anyCollectionOf;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+@SuppressWarnings({"UnusedDeclaration", "StatementWithEmptyBody"})
+public class TestTSOChannelHandlerNetty {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestTSOChannelHandlerNetty.class);
+
+ @Mock
+ private
+ RequestProcessor requestProcessor;
+
+ // Component under test
+ private TSOChannelHandler channelHandler;
+
+ @BeforeMethod
+ public void beforeTestMethod() {
+ MockitoAnnotations.initMocks(this);
+ TSOServerConfig config = new TSOServerConfig();
+ config.setPort(1434);
+ channelHandler = new TSOChannelHandler(config, requestProcessor, new NullMetricsProvider());
+ }
+
+ @AfterMethod
+ public void afterTestMethod() throws IOException {
+ channelHandler.close();
+ }
+
+ @Test(timeOut = 10_000)
+ public void testMainAPI() throws Exception {
+
+ // Check initial state
+ assertNull(channelHandler.listeningChannel);
+ assertNull(channelHandler.channelGroup);
+
+ // Check initial connection
+ channelHandler.reconnect();
+ assertTrue(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 1);
+ assertEquals(((InetSocketAddress) channelHandler.listeningChannel.getLocalAddress()).getPort(), 1434);
+
+ // Check connection close
+ channelHandler.closeConnection();
+ assertFalse(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 0);
+
+ // Check re-closing connection
+ channelHandler.closeConnection();
+ assertFalse(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 0);
+
+ // Check connection after closing
+ channelHandler.reconnect();
+ assertTrue(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 1);
+
+ // Check re-connection
+ channelHandler.reconnect();
+ assertTrue(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 1);
+
+ // Exercise closeable with re-connection trial
+ channelHandler.close();
+ assertFalse(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 0);
+ try {
+ channelHandler.reconnect();
+ } catch (ChannelException e) {
+ // Expected: Can't reconnect after closing
+ assertFalse(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 0);
+ }
+
+ }
+
+ @Test(timeOut = 10_000)
+ public void testNettyConnectionToTSOFromClient() throws Exception {
+
+ ClientBootstrap nettyClient = createNettyClientBootstrap();
+
+ ChannelFuture channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Test the client can't connect cause the server is not there
+ // ------------------------------------------------------------------------------------------------------------
+ while (!channelF.isDone()) /** do nothing */ ;
+ assertFalse(channelF.isSuccess());
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Test creation of a server connection
+ // ------------------------------------------------------------------------------------------------------------
+ channelHandler.reconnect();
+ assertTrue(channelHandler.listeningChannel.isOpen());
+ // Eventually the channel group of the server should contain the listening channel
+ assertEquals(channelHandler.channelGroup.size(), 1);
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Test that a client can connect now
+ // ------------------------------------------------------------------------------------------------------------
+ channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
+ while (!channelF.isDone()) /** do nothing */ ;
+ assertTrue(channelF.isSuccess());
+ assertTrue(channelF.getChannel().isConnected());
+ // Eventually the channel group of the server should have 2 elements
+ while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Close the channel on the client side and test we have one element less in the channel group
+ // ------------------------------------------------------------------------------------------------------------
+ channelF.getChannel().close().await();
+ // Eventually the channel group of the server should have only one element
+ while (channelHandler.channelGroup.size() != 1) /** do nothing */ ;
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Open a new channel and test the connection closing on the server side through the channel handler
+ // ------------------------------------------------------------------------------------------------------------
+ channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
+ while (!channelF.isDone()) /** do nothing */ ;
+ assertTrue(channelF.isSuccess());
+ // Eventually the channel group of the server should have 2 elements again
+ while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
+ channelHandler.closeConnection();
+ assertFalse(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 0);
+ // Wait some time and check the channel was closed
+ TimeUnit.SECONDS.sleep(1);
+ assertFalse(channelF.getChannel().isOpen());
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Test server re-connections with connected clients
+ // ------------------------------------------------------------------------------------------------------------
+ // Connect first time
+ channelHandler.reconnect();
+ assertTrue(channelHandler.listeningChannel.isOpen());
+ // Eventually the channel group of the server should contain the listening channel
+ assertEquals(channelHandler.channelGroup.size(), 1);
+ // Check the client can connect
+ channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
+ while (!channelF.isDone()) /** do nothing */ ;
+ assertTrue(channelF.isSuccess());
+ // Eventually the channel group of the server should have 2 elements
+ while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
+ // Re-connect and check that client connection was gone
+ channelHandler.reconnect();
+ assertTrue(channelHandler.listeningChannel.isOpen());
+ // Eventually the channel group of the server should contain the listening channel
+ assertEquals(channelHandler.channelGroup.size(), 1);
+ // Wait some time and check the channel was closed
+ TimeUnit.SECONDS.sleep(1);
+ assertFalse(channelF.getChannel().isOpen());
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Test closeable interface with re-connection trial
+ // ------------------------------------------------------------------------------------------------------------
+ channelHandler.close();
+ assertFalse(channelHandler.listeningChannel.isOpen());
+ assertEquals(channelHandler.channelGroup.size(), 0);
+ }
+
+ @Test(timeOut = 10_000)
+ public void testNettyChannelWriting() throws Exception {
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Prepare test
+ // ------------------------------------------------------------------------------------------------------------
+
+ // Connect channel handler
+ channelHandler.reconnect();
+ // Create client and connect it
+ ClientBootstrap nettyClient = createNettyClientBootstrap();
+ ChannelFuture channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
+ // Basic checks for connection
+ while (!channelF.isDone()) /** do nothing */ ;
+ assertTrue(channelF.isSuccess());
+ assertTrue(channelF.getChannel().isConnected());
+ Channel channel = channelF.getChannel();
+ // Eventually the channel group of the server should have 2 elements
+ while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
+ // Write first handshake request
+ TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
+ // NOTE: Add here the required handshake capabilities when necessary
+ handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
+ channelF.getChannel().write(TSOProto.Request.newBuilder().setHandshakeRequest(handshake.build()).build());
+
+ // ------------------------------------------------------------------------------------------------------------
+ // Test channel writing
+ // ------------------------------------------------------------------------------------------------------------
+ testWritingTimestampRequest(channel);
+
+ testWritingCommitRequest(channel);
+ }
+
+ private void testWritingTimestampRequest(Channel channel) throws InterruptedException {
+ // Reset mock
+ reset(requestProcessor);
+ TSOProto.Request.Builder tsBuilder = TSOProto.Request.newBuilder();
+ TSOProto.TimestampRequest.Builder tsRequestBuilder = TSOProto.TimestampRequest.newBuilder();
+ tsBuilder.setTimestampRequest(tsRequestBuilder.build());
+ // Write into the channel
+ channel.write(tsBuilder.build()).await();
+ verify(requestProcessor, timeout(100).times(1)).timestampRequest(any(Channel.class), any(MonitoringContext.class));
+ verify(requestProcessor, timeout(100).never())
+ .commitRequest(anyLong(), anyCollectionOf(Long.class), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
+ }
+
+ private void testWritingCommitRequest(Channel channel) throws InterruptedException {
+ // Reset mock
+ reset(requestProcessor);
+ TSOProto.Request.Builder commitBuilder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitRequestBuilder = TSOProto.CommitRequest.newBuilder();
+ commitRequestBuilder.setStartTimestamp(666);
+ commitRequestBuilder.addCellId(666);
+ commitBuilder.setCommitRequest(commitRequestBuilder.build());
+ TSOProto.Request r = commitBuilder.build();
+ assertTrue(r.hasCommitRequest());
+ // Write into the channel
+ channel.write(commitBuilder.build()).await();
+ verify(requestProcessor, timeout(100).never()).timestampRequest(any(Channel.class), any(MonitoringContext.class));
+ verify(requestProcessor, timeout(100).times(1))
+ .commitRequest(eq(666L), anyCollectionOf(Long.class), eq(false), any(Channel.class), any(MonitoringContext.class));
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private ClientBootstrap createNettyClientBootstrap() {
+
+ ChannelFactory factory = new NioClientSocketChannelFactory(
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("client-boss-%d").build()),
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("client-worker-%d").build()), 1);
+ // Create the bootstrap
+ ClientBootstrap bootstrap = new ClientBootstrap(factory);
+ bootstrap.setOption("tcpNoDelay", true);
+ bootstrap.setOption("keepAlive", true);
+ bootstrap.setOption("reuseAddress", true);
+ bootstrap.setOption("connectTimeoutMillis", 100);
+ ChannelPipeline pipeline = bootstrap.getPipeline();
+ pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
+ pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+ pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
+ pipeline.addLast("protobufencoder", new ProtobufEncoder());
+ pipeline.addLast("handler", new SimpleChannelHandler() {
+
+ @Override
+ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
+ LOG.info("Channel {} connected", ctx.getChannel());
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+ LOG.error("Error on channel {}", ctx.getChannel(), e.getCause());
+ }
+
+ });
+ return bootstrap;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/org/apache/omid/tso/TestTSOStateManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/org/apache/omid/tso/TestTSOStateManager.java b/tso-server/src/test/java/org/apache/omid/tso/TestTSOStateManager.java
new file mode 100644
index 0000000..350db5f
--- /dev/null
+++ b/tso-server/src/test/java/org/apache/omid/tso/TestTSOStateManager.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso;
+
+import org.apache.omid.tso.TSOStateManager.StateObserver;
+import org.apache.omid.tso.TSOStateManager.TSOState;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class TestTSOStateManager {
+
+ private static final long INITIAL_STATE_VALUE = 1L;
+ private static final long NEW_STATE_VALUE = 1000;
+
+ // Mocks
+ private TimestampOracle timestampOracle = mock(TimestampOracle.class);
+
+ // Component under test
+ private TSOStateManager stateManager = new TSOStateManagerImpl(timestampOracle);
+
+ @BeforeMethod
+ public void beforeMethod() {
+ // Initialize the state with the one reported by the Timestamp Oracle
+ when(timestampOracle.getLast()).thenReturn(INITIAL_STATE_VALUE);
+ }
+
+ @Test
+ public void testResetOfTSOServerState() throws Exception {
+
+ // Reset the state and check we get the initial state values
+ TSOState initialState = stateManager.reset();
+ assertEquals(initialState.getLowWatermark(), INITIAL_STATE_VALUE);
+ assertEquals(initialState.getEpoch(), INITIAL_STATE_VALUE);
+ assertTrue("In this implementation low watermark should be equal to epoch",
+ initialState.getLowWatermark() == initialState.getEpoch());
+
+ // Then, simulate a change in the state returned by the Timestamp Oracle...
+ when(timestampOracle.getLast()).thenReturn(NEW_STATE_VALUE);
+ // ... and again, reset the state and check we get the new values
+ TSOState secondState = stateManager.reset();
+ assertEquals(secondState.getLowWatermark(), NEW_STATE_VALUE);
+ assertEquals(secondState.getEpoch(), NEW_STATE_VALUE);
+ assertTrue("In this implementation low watermark should be equal to epoch",
+ secondState.getLowWatermark() == secondState.getEpoch());
+
+ }
+
+ @Test
+ public void testObserverRegistrationAndDeregistrationForStateChanges() throws Exception {
+
+ // Register observer 1 for receiving state changes
+ StateObserver observer1 = spy(new DummyObserver());
+ stateManager.register(observer1);
+
+ // Reset the state to trigger observer notifications
+ TSOState state = stateManager.reset();
+
+ // Check observer 1 was notified with the corresponding state
+ verify(observer1, timeout(100).times(1)).update(eq(state));
+
+ // Register observer 1 for receiving state changes
+ StateObserver observer2 = spy(new DummyObserver());
+ stateManager.register(observer2);
+
+ // Again, reset the state to trigger observer notifications
+ state = stateManager.reset();
+
+ // Check both observers were notified with the corresponding state
+ verify(observer1, timeout(100).times(1)).update(eq(state));
+ verify(observer2, timeout(100).times(1)).update(eq(state));
+
+ // De-register observer 1
+ stateManager.unregister(observer1);
+
+ // Again, reset the state to trigger observer notifications
+ state = stateManager.reset();
+
+ // Check only observer 2 was notified
+ verify(observer1, timeout(100).times(0)).update(eq(state));
+ verify(observer2, timeout(100).times(1)).update(eq(state));
+ }
+
+ // ------------------------------------------------------------------------
+ // -------------------------- Helper classes ------------------------------
+ // ------------------------------------------------------------------------
+
+ private class DummyObserver implements StateObserver {
+
+ @Override
+ public void update(TSOState state) throws IOException {
+ }
+
+ }
+
+}
[31/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java b/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
deleted file mode 100644
index 674cdd7..0000000
--- a/hbase-common/src/test/java/com/yahoo/omid/committable/hbase/TestHBaseCommitTableKey.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.committable.hbase;
-
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.BadRandomKeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.BucketKeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.FullRandomKeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.SeqKeyGenerator;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-
-import static java.lang.Long.MAX_VALUE;
-import static org.testng.Assert.assertEquals;
-
-public class TestHBaseCommitTableKey {
-
- @Test
- public void testEncodeDecode() throws Exception {
- testKeyGen(new BucketKeyGenerator());
- testKeyGen(new BadRandomKeyGenerator());
- testKeyGen(new FullRandomKeyGenerator());
- testKeyGen(new SeqKeyGenerator());
- }
-
- @Test(enabled = false)
- private void testKeyGen(KeyGenerator keyGen) throws IOException {
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(0)), 0, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1)), 1, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(8)), 8, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1024)), 1024, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1234)), 1234, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(4321)), 4321, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(0xdeadbeefcafeL)), 0xdeadbeefcafeL, "Should match");
- assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(MAX_VALUE)), MAX_VALUE, "Should match");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-common/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTableKey.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTableKey.java b/hbase-common/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTableKey.java
new file mode 100644
index 0000000..f6b6877
--- /dev/null
+++ b/hbase-common/src/test/java/org/apache/omid/committable/hbase/TestHBaseCommitTableKey.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.committable.hbase;
+
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.BadRandomKeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.BucketKeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.FullRandomKeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations.SeqKeyGenerator;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static java.lang.Long.MAX_VALUE;
+import static org.testng.Assert.assertEquals;
+
+public class TestHBaseCommitTableKey {
+
+ @Test
+ public void testEncodeDecode() throws Exception {
+ testKeyGen(new BucketKeyGenerator());
+ testKeyGen(new BadRandomKeyGenerator());
+ testKeyGen(new FullRandomKeyGenerator());
+ testKeyGen(new SeqKeyGenerator());
+ }
+
+ @Test(enabled = false)
+ private void testKeyGen(KeyGenerator keyGen) throws IOException {
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(0)), 0, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1)), 1, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(8)), 8, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1024)), 1024, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(1234)), 1234, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(4321)), 4321, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(0xdeadbeefcafeL)), 0xdeadbeefcafeL, "Should match");
+ assertEquals(keyGen.keyToStartTimestamp(keyGen.startTimestampToKey(MAX_VALUE)), MAX_VALUE, "Should match");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index 8ac01a6..ffc16c5 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -17,24 +17,24 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>tso-server</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-client</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>tso-server</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
@@ -123,7 +123,7 @@
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<artifactSet>
<includes>
- <include>com.yahoo.omid:*</include>
+ <include>org.apache.omid:*</include>
</includes>
</artifactSet>
<finalName>omid-hbase-coprocessor-${project.version}</finalName>
@@ -143,7 +143,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -154,7 +154,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
deleted file mode 100644
index e391b2b..0000000
--- a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/CompactorUtil.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.yahoo.omid.tools.hbase.HBaseLogin;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-
-public class CompactorUtil {
-
- public static void enableOmidCompaction(Configuration conf,
- TableName table, byte[] columnFamily) throws IOException {
- HBaseAdmin admin = new HBaseAdmin(conf);
- try {
- HTableDescriptor desc = admin.getTableDescriptor(table);
- HColumnDescriptor cfDesc = desc.getFamily(columnFamily);
- cfDesc.setValue(OmidCompactor.OMID_COMPACTABLE_CF_FLAG,
- Boolean.TRUE.toString());
- admin.modifyColumn(table, cfDesc);
- } finally {
- admin.close();
- }
- }
-
- public static void disableOmidCompaction(Configuration conf,
- TableName table, byte[] columnFamily) throws IOException {
- HBaseAdmin admin = new HBaseAdmin(conf);
- try {
- HTableDescriptor desc = admin.getTableDescriptor(table);
- HColumnDescriptor cfDesc = desc.getFamily(columnFamily);
- cfDesc.setValue(OmidCompactor.OMID_COMPACTABLE_CF_FLAG,
- Boolean.FALSE.toString());
- admin.modifyColumn(table, cfDesc);
- } finally {
- admin.close();
- }
- }
-
- static class Config {
- @Parameter(names = "-table", required = true)
- String table;
-
- @Parameter(names = "-columnFamily", required = false)
- String columnFamily;
-
- @Parameter(names = "-help")
- boolean help = false;
-
- @Parameter(names = "-enable")
- boolean enable = false;
-
- @Parameter(names = "-disable")
- boolean disable = false;
-
- @ParametersDelegate
- private SecureHBaseConfig loginFlags = new SecureHBaseConfig();
-
- }
-
- public static void main(String[] args) throws IOException {
- Config cmdline = new Config();
- JCommander jcommander = new JCommander(cmdline, args);
- if (cmdline.help) {
- jcommander.usage("CompactorUtil");
- System.exit(1);
- }
-
- HBaseLogin.loginIfNeeded(cmdline.loginFlags);
-
- Configuration conf = HBaseConfiguration.create();
- if (cmdline.enable) {
- enableOmidCompaction(conf, TableName.valueOf(cmdline.table),
- Bytes.toBytes(cmdline.columnFamily));
- } else if (cmdline.disable) {
- disableOmidCompaction(conf, TableName.valueOf(cmdline.table),
- Bytes.toBytes(cmdline.columnFamily));
- } else {
- System.err.println("Must specify enable or disable");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
deleted file mode 100644
index e3e181e..0000000
--- a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/HRegionCellGetterAdapter.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.yahoo.omid.transaction.CellUtils.CellGetter;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-
-import java.io.IOException;
-
-public class HRegionCellGetterAdapter implements CellGetter {
-
- private final HRegion hRegion;
-
- public HRegionCellGetterAdapter(HRegion hRegion) {
- this.hRegion = hRegion;
- }
-
- public Result get(Get get) throws IOException {
- return hRegion.get(get);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java b/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
deleted file mode 100644
index 8ab18b6..0000000
--- a/hbase-coprocessor/src/main/java/com/yahoo/omid/transaction/OmidCompactor.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.CoprocessorEnvironment;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
-import org.apache.hadoop.hbase.coprocessor.ObserverContext;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.CompactorScanner;
-import org.apache.hadoop.hbase.regionserver.InternalScanner;
-import org.apache.hadoop.hbase.regionserver.ScanType;
-import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentLinkedQueue;
-
-import static com.yahoo.omid.committable.hbase.HBaseCommitTableConfig.COMMIT_TABLE_NAME_KEY;
-
-/**
- * Garbage collector for stale data: triggered upon HBase
- * compactions, it removes data from uncommitted transactions
- * older than the low watermark using a special scanner
- */
-public class OmidCompactor extends BaseRegionObserver {
-
- private static final Logger LOG = LoggerFactory.getLogger(OmidCompactor.class);
-
- private static final String HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_KEY
- = "omid.hbase.compactor.retain.tombstones";
- private static final boolean HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_DEFAULT = true;
-
- final static String OMID_COMPACTABLE_CF_FLAG = "OMID_ENABLED";
-
- private HBaseCommitTableConfig commitTableConf = null;
- private Configuration conf = null;
- @VisibleForTesting
- Queue<CommitTable.Client> commitTableClientQueue = new ConcurrentLinkedQueue<>();
-
- // When compacting, if a cell which has been marked by HBase as Delete or
- // Delete Family (that is, non-transactionally deleted), we allow the user
- // to decide what the compactor scanner should do with it: retain it or not
- // If retained, the deleted cell will appear after a minor compaction, but
- // will be deleted anyways after a major one
- private boolean retainNonTransactionallyDeletedCells;
-
- public OmidCompactor() {
- LOG.info("Compactor coprocessor initialized via empty constructor");
- }
-
- @Override
- public void start(CoprocessorEnvironment env) throws IOException {
- LOG.info("Starting compactor coprocessor");
- conf = env.getConfiguration();
- commitTableConf = new HBaseCommitTableConfig();
- String commitTableName = conf.get(COMMIT_TABLE_NAME_KEY);
- if (commitTableName != null) {
- commitTableConf.setTableName(commitTableName);
- }
- retainNonTransactionallyDeletedCells =
- conf.getBoolean(HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_KEY,
- HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_DEFAULT);
- LOG.info("Compactor coprocessor started");
- }
-
- @Override
- public void stop(CoprocessorEnvironment e) throws IOException {
- LOG.info("Stopping compactor coprocessor");
- if (commitTableClientQueue != null) {
- for (CommitTable.Client commitTableClient : commitTableClientQueue) {
- commitTableClient.close();
- }
- }
- LOG.info("Compactor coprocessor stopped");
- }
-
- @Override
- public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
- Store store,
- InternalScanner scanner,
- ScanType scanType,
- CompactionRequest request) throws IOException {
- HTableDescriptor desc = e.getEnvironment().getRegion().getTableDesc();
- HColumnDescriptor famDesc
- = desc.getFamily(Bytes.toBytes(store.getColumnFamilyName()));
- boolean omidCompactable = Boolean.valueOf(famDesc.getValue(OMID_COMPACTABLE_CF_FLAG));
- // only column families tagged as compactable are compacted
- // with omid compactor
- if (!omidCompactable) {
- return scanner;
- } else {
- CommitTable.Client commitTableClient = commitTableClientQueue.poll();
- if (commitTableClient == null) {
- commitTableClient = initAndGetCommitTableClient();
- }
- boolean isMajorCompaction = request.isMajor();
- return new CompactorScanner(e,
- scanner,
- commitTableClient,
- commitTableClientQueue,
- isMajorCompaction,
- retainNonTransactionallyDeletedCells);
- }
- }
-
- private CommitTable.Client initAndGetCommitTableClient() throws IOException {
- LOG.info("Trying to get the commit table client");
- CommitTable commitTable = new HBaseCommitTable(conf, commitTableConf);
- CommitTable.Client commitTableClient = commitTable.getClient();
- LOG.info("Commit table client obtained {}", commitTableClient.getClass().getCanonicalName());
- return commitTableClient;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java b/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
index 17075f9..55e9db2 100644
--- a/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
+++ b/hbase-coprocessor/src/main/java/org/apache/hadoop/hbase/regionserver/CompactorScanner.java
@@ -21,12 +21,12 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
-import com.yahoo.omid.HBaseShims;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.Client;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.transaction.CellUtils;
-import com.yahoo.omid.transaction.CellInfo;
+import org.apache.omid.HBaseShims;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.Client;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.transaction.CellUtils;
+import org.apache.omid.transaction.CellInfo;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
@@ -48,7 +48,7 @@ import java.util.Queue;
import java.util.SortedMap;
import java.util.concurrent.ExecutionException;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.SHADOW_CELL;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.SHADOW_CELL;
public class CompactorScanner implements InternalScanner {
private static final Logger LOG = LoggerFactory.getLogger(CompactorScanner.class);
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/org/apache/omid/transaction/CompactorUtil.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/org/apache/omid/transaction/CompactorUtil.java b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/CompactorUtil.java
new file mode 100644
index 0000000..e402916
--- /dev/null
+++ b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/CompactorUtil.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParametersDelegate;
+import org.apache.omid.tools.hbase.HBaseLogin;
+import org.apache.omid.tools.hbase.SecureHBaseConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+
+import java.io.IOException;
+
+public class CompactorUtil {
+
+ public static void enableOmidCompaction(Configuration conf,
+ TableName table, byte[] columnFamily) throws IOException {
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ try {
+ HTableDescriptor desc = admin.getTableDescriptor(table);
+ HColumnDescriptor cfDesc = desc.getFamily(columnFamily);
+ cfDesc.setValue(OmidCompactor.OMID_COMPACTABLE_CF_FLAG,
+ Boolean.TRUE.toString());
+ admin.modifyColumn(table, cfDesc);
+ } finally {
+ admin.close();
+ }
+ }
+
+ public static void disableOmidCompaction(Configuration conf,
+ TableName table, byte[] columnFamily) throws IOException {
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ try {
+ HTableDescriptor desc = admin.getTableDescriptor(table);
+ HColumnDescriptor cfDesc = desc.getFamily(columnFamily);
+ cfDesc.setValue(OmidCompactor.OMID_COMPACTABLE_CF_FLAG,
+ Boolean.FALSE.toString());
+ admin.modifyColumn(table, cfDesc);
+ } finally {
+ admin.close();
+ }
+ }
+
+ static class Config {
+ @Parameter(names = "-table", required = true)
+ String table;
+
+ @Parameter(names = "-columnFamily", required = false)
+ String columnFamily;
+
+ @Parameter(names = "-help")
+ boolean help = false;
+
+ @Parameter(names = "-enable")
+ boolean enable = false;
+
+ @Parameter(names = "-disable")
+ boolean disable = false;
+
+ @ParametersDelegate
+ private SecureHBaseConfig loginFlags = new SecureHBaseConfig();
+
+ }
+
+ public static void main(String[] args) throws IOException {
+ Config cmdline = new Config();
+ JCommander jcommander = new JCommander(cmdline, args);
+ if (cmdline.help) {
+ jcommander.usage("CompactorUtil");
+ System.exit(1);
+ }
+
+ HBaseLogin.loginIfNeeded(cmdline.loginFlags);
+
+ Configuration conf = HBaseConfiguration.create();
+ if (cmdline.enable) {
+ enableOmidCompaction(conf, TableName.valueOf(cmdline.table),
+ Bytes.toBytes(cmdline.columnFamily));
+ } else if (cmdline.disable) {
+ disableOmidCompaction(conf, TableName.valueOf(cmdline.table),
+ Bytes.toBytes(cmdline.columnFamily));
+ } else {
+ System.err.println("Must specify enable or disable");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/org/apache/omid/transaction/HRegionCellGetterAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/org/apache/omid/transaction/HRegionCellGetterAdapter.java b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/HRegionCellGetterAdapter.java
new file mode 100644
index 0000000..c5676e6
--- /dev/null
+++ b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/HRegionCellGetterAdapter.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.omid.transaction.CellUtils.CellGetter;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+
+import java.io.IOException;
+
+public class HRegionCellGetterAdapter implements CellGetter {
+
+ private final HRegion hRegion;
+
+ public HRegionCellGetterAdapter(HRegion hRegion) {
+ this.hRegion = hRegion;
+ }
+
+ public Result get(Get get) throws IOException {
+ return hRegion.get(get);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/main/java/org/apache/omid/transaction/OmidCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/main/java/org/apache/omid/transaction/OmidCompactor.java b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/OmidCompactor.java
new file mode 100644
index 0000000..71ec909
--- /dev/null
+++ b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/OmidCompactor.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.CompactorScanner;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import static org.apache.omid.committable.hbase.HBaseCommitTableConfig.COMMIT_TABLE_NAME_KEY;
+
+/**
+ * Garbage collector for stale data: triggered upon HBase
+ * compactions, it removes data from uncommitted transactions
+ * older than the low watermark using a special scanner
+ */
+public class OmidCompactor extends BaseRegionObserver {
+
+ private static final Logger LOG = LoggerFactory.getLogger(OmidCompactor.class);
+
+ private static final String HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_KEY
+ = "omid.hbase.compactor.retain.tombstones";
+ private static final boolean HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_DEFAULT = true;
+
+ final static String OMID_COMPACTABLE_CF_FLAG = "OMID_ENABLED";
+
+ private HBaseCommitTableConfig commitTableConf = null;
+ private Configuration conf = null;
+ @VisibleForTesting
+ Queue<CommitTable.Client> commitTableClientQueue = new ConcurrentLinkedQueue<>();
+
+ // When compacting, if a cell which has been marked by HBase as Delete or
+ // Delete Family (that is, non-transactionally deleted), we allow the user
+ // to decide what the compactor scanner should do with it: retain it or not
+ // If retained, the deleted cell will appear after a minor compaction, but
+ // will be deleted anyways after a major one
+ private boolean retainNonTransactionallyDeletedCells;
+
+ public OmidCompactor() {
+ LOG.info("Compactor coprocessor initialized via empty constructor");
+ }
+
+ @Override
+ public void start(CoprocessorEnvironment env) throws IOException {
+ LOG.info("Starting compactor coprocessor");
+ conf = env.getConfiguration();
+ commitTableConf = new HBaseCommitTableConfig();
+ String commitTableName = conf.get(COMMIT_TABLE_NAME_KEY);
+ if (commitTableName != null) {
+ commitTableConf.setTableName(commitTableName);
+ }
+ retainNonTransactionallyDeletedCells =
+ conf.getBoolean(HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_KEY,
+ HBASE_RETAIN_NON_TRANSACTIONALLY_DELETED_CELLS_DEFAULT);
+ LOG.info("Compactor coprocessor started");
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment e) throws IOException {
+ LOG.info("Stopping compactor coprocessor");
+ if (commitTableClientQueue != null) {
+ for (CommitTable.Client commitTableClient : commitTableClientQueue) {
+ commitTableClient.close();
+ }
+ }
+ LOG.info("Compactor coprocessor stopped");
+ }
+
+ @Override
+ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
+ Store store,
+ InternalScanner scanner,
+ ScanType scanType,
+ CompactionRequest request) throws IOException {
+ HTableDescriptor desc = e.getEnvironment().getRegion().getTableDesc();
+ HColumnDescriptor famDesc
+ = desc.getFamily(Bytes.toBytes(store.getColumnFamilyName()));
+ boolean omidCompactable = Boolean.valueOf(famDesc.getValue(OMID_COMPACTABLE_CF_FLAG));
+ // only column families tagged as compactable are compacted
+ // with omid compactor
+ if (!omidCompactable) {
+ return scanner;
+ } else {
+ CommitTable.Client commitTableClient = commitTableClientQueue.poll();
+ if (commitTableClient == null) {
+ commitTableClient = initAndGetCommitTableClient();
+ }
+ boolean isMajorCompaction = request.isMajor();
+ return new CompactorScanner(e,
+ scanner,
+ commitTableClient,
+ commitTableClientQueue,
+ isMajorCompaction,
+ retainNonTransactionallyDeletedCells);
+ }
+ }
+
+ private CommitTable.Client initAndGetCommitTableClient() throws IOException {
+ LOG.info("Trying to get the commit table client");
+ CommitTable commitTable = new HBaseCommitTable(conf, commitTableConf);
+ CommitTable.Client commitTableClient = commitTable.getClient();
+ LOG.info("Commit table client obtained {}", commitTableClient.getClass().getCanonicalName());
+ return commitTableClient;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java b/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
deleted file mode 100644
index 6d0f519..0000000
--- a/hbase-coprocessor/src/test/java/com/yahoo/omid/transaction/TSOForHBaseCompactorTestModule.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorage;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import com.yahoo.omid.tso.DisruptorModule;
-import com.yahoo.omid.tso.LeaseManagement;
-import com.yahoo.omid.tso.MockPanicker;
-import com.yahoo.omid.tso.NetworkInterfaceUtils;
-import com.yahoo.omid.tso.Panicker;
-import com.yahoo.omid.tso.TSOChannelHandler;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.TSOStateManager;
-import com.yahoo.omid.tso.TSOStateManagerImpl;
-import com.yahoo.omid.tso.TimestampOracle;
-import com.yahoo.omid.tso.TimestampOracleImpl;
-import com.yahoo.omid.tso.VoidLeaseManager;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-import java.io.File;
-import java.io.IOException;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-class TSOForHBaseCompactorTestModule extends AbstractModule {
-
- private final TSOServerConfig config;
-
- TSOForHBaseCompactorTestModule(TSOServerConfig config) {
- this.config = config;
- }
-
- @Override
- protected void configure() {
-
- bind(TSOChannelHandler.class).in(Singleton.class);
-
- bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
-
- bind(Panicker.class).to(MockPanicker.class);
- // HBase commit table creation
- bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class);
- // Timestamp storage creation
- bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class);
- bind(TimestampOracle.class).to(TimestampOracleImpl.class).in(Singleton.class);
-
- // DisruptorConfig
- install(new DisruptorModule());
-
- }
-
- @Provides
- @Singleton
- Configuration provideHBaseConfig() throws IOException {
- Configuration hbaseConf = HBaseConfiguration.create();
- hbaseConf.setInt("hbase.hregion.memstore.flush.size", 10_000 * 1024);
- hbaseConf.setInt("hbase.regionserver.nbreservationblocks", 1);
- hbaseConf.set("tso.host", "localhost");
- hbaseConf.setInt("tso.port", 1234);
- hbaseConf.set("hbase.coprocessor.region.classes", "com.yahoo.omid.transaction.OmidCompactor");
- final String rootdir = "/tmp/hbase.test.dir/";
- File rootdirFile = new File(rootdir);
- FileUtils.deleteDirectory(rootdirFile);
- hbaseConf.set("hbase.rootdir", rootdir);
- return hbaseConf;
- }
-
- @Provides
- TSOServerConfig provideTSOServerConfig() {
- return config;
- }
-
- @Provides
- @Singleton
- MetricsRegistry provideMetricsRegistry() {
- return new NullMetricsProvider();
- }
-
- @Provides
- @Singleton
- LeaseManagement provideLeaseManager(TSOChannelHandler tsoChannelHandler,
- TSOStateManager stateManager) throws IOException {
- return new VoidLeaseManager(tsoChannelHandler, stateManager);
- }
-
- @Provides
- @Named(TSO_HOST_AND_PORT_KEY)
- String provideTSOHostAndPort() throws SocketException, UnknownHostException {
- return NetworkInterfaceUtils.getTSOHostAndPort(config);
-
- }
-}
\ No newline at end of file
[44/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
Move com.yahoo -> org.apache
Change-Id: Ic3fbf788f932f5083ab99e5c1bcf9a69cb70ed41
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/9cd856c6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/9cd856c6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/9cd856c6
Branch: refs/heads/master
Commit: 9cd856c67f6e9ccd4f52ffbfd8cf4b483691654a
Parents: 5734cac
Author: Igor Katkov <ka...@yahoo-inc.com>
Authored: Fri Apr 15 16:05:25 2016 -0700
Committer: Igor Katkov <ka...@yahoo-inc.com>
Committed: Fri Apr 15 16:09:01 2016 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
benchmarks/bin/benchmarks.sh | 2 +-
benchmarks/pom.xml | 10 +-
.../hbase/HBaseCommitTableTester.java | 137 --
.../yahoo/omid/benchmarks/tso/RawTxRunner.java | 301 ----
.../omid/benchmarks/tso/TSOServerBenchmark.java | 138 --
.../tso/TSOServerBenchmarkConfig.java | 160 ---
.../yahoo/omid/benchmarks/utils/Generator.java | 36 -
.../omid/benchmarks/utils/IntegerGenerator.java | 71 -
.../utils/ScrambledZipfianGenerator.java | 153 ---
.../omid/benchmarks/utils/UniformGenerator.java | 36 -
.../omid/benchmarks/utils/ZipfianGenerator.java | 298 ----
.../hbase/HBaseCommitTableTester.java | 137 ++
.../apache/omid/benchmarks/tso/RawTxRunner.java | 301 ++++
.../omid/benchmarks/tso/TSOServerBenchmark.java | 138 ++
.../tso/TSOServerBenchmarkConfig.java | 160 +++
.../apache/omid/benchmarks/utils/Generator.java | 30 +
.../omid/benchmarks/utils/IntegerGenerator.java | 65 +
.../utils/ScrambledZipfianGenerator.java | 147 ++
.../omid/benchmarks/utils/UniformGenerator.java | 36 +
.../omid/benchmarks/utils/ZipfianGenerator.java | 292 ++++
.../default-tso-server-benchmark-config.yml | 20 +-
codahale-metrics/pom.xml | 4 +-
.../omid/metrics/CodahaleMetricsConfig.java | 100 --
.../omid/metrics/CodahaleMetricsProvider.java | 346 -----
.../omid/metrics/CodahaleMetricsConfig.java | 100 ++
.../omid/metrics/CodahaleMetricsProvider.java | 346 +++++
commit-table/pom.xml | 2 +-
.../com/yahoo/omid/committable/CommitTable.java | 113 --
.../omid/committable/InMemoryCommitTable.java | 143 --
.../yahoo/omid/committable/NullCommitTable.java | 91 --
.../apache/omid/committable/CommitTable.java | 113 ++
.../omid/committable/InMemoryCommitTable.java | 143 ++
.../omid/committable/NullCommitTable.java | 91 ++
.../omid/committable/NullCommitTableTest.java | 70 -
.../omid/committable/NullCommitTableTest.java | 70 +
common/pom.xml | 2 +-
.../src/main/java/com/yahoo/omid/YAMLUtils.java | 81 --
.../main/java/com/yahoo/omid/zk/ZKUtils.java | 57 -
.../main/java/org/apache/omid/YAMLUtils.java | 81 ++
.../main/java/org/apache/omid/zk/ZKUtils.java | 57 +
common/src/main/proto/TSOProto.proto | 2 +-
.../test/java/com/yahoo/omid/YAMLUtilsTest.java | 50 -
.../java/org/apache/omid/YAMLUtilsTest.java | 50 +
examples/pom.xml | 10 +-
examples/run.sh | 6 +-
.../com/yahoo/omid/examples/BasicExample.java | 112 --
.../omid/examples/ConfigurationExample.java | 131 --
.../omid/examples/SnapshotIsolationExample.java | 174 ---
.../org/apache/omid/examples/BasicExample.java | 112 ++
.../omid/examples/ConfigurationExample.java | 131 ++
.../omid/examples/SnapshotIsolationExample.java | 174 +++
.../main/resources/hbase-omid-client-config.yml | 10 +-
examples/src/main/resources/log4j.properties | 3 +-
hbase-client/pom.xml | 18 +-
.../yahoo/omid/transaction/ColumnWrapper.java | 64 -
.../transaction/HBaseAsyncPostCommitter.java | 67 -
.../com/yahoo/omid/transaction/HBaseCellId.java | 80 --
.../HBaseOmidClientConfiguration.java | 170 ---
.../transaction/HBaseSyncPostCommitter.java | 125 --
.../omid/transaction/HBaseTransaction.java | 79 --
.../transaction/HBaseTransactionClient.java | 24 -
.../transaction/HBaseTransactionManager.java | 292 ----
.../java/com/yahoo/omid/transaction/TTable.java | 770 -----------
.../transaction/TTableCellGetterAdapter.java | 38 -
.../apache/omid/transaction/ColumnWrapper.java | 64 +
.../transaction/HBaseAsyncPostCommitter.java | 67 +
.../apache/omid/transaction/HBaseCellId.java | 80 ++
.../HBaseOmidClientConfiguration.java | 170 +++
.../transaction/HBaseSyncPostCommitter.java | 125 ++
.../omid/transaction/HBaseTransaction.java | 79 ++
.../transaction/HBaseTransactionClient.java | 24 +
.../transaction/HBaseTransactionManager.java | 292 ++++
.../org/apache/omid/transaction/TTable.java | 770 +++++++++++
.../transaction/TTableCellGetterAdapter.java | 38 +
.../default-hbase-omid-client-config.yml | 4 +-
.../yahoo/omid/transaction/OmidTestBase.java | 257 ----
.../TestAsynchronousPostCommitter.java | 323 -----
.../yahoo/omid/transaction/TestAutoFlush.java | 62 -
.../omid/transaction/TestBasicTransaction.java | 440 ------
.../yahoo/omid/transaction/TestCellUtils.java | 230 ----
.../omid/transaction/TestColumnIterator.java | 96 --
.../yahoo/omid/transaction/TestDeletion.java | 299 ----
.../TestEndToEndScenariosWithHA.java | 412 ------
.../com/yahoo/omid/transaction/TestFilters.java | 191 ---
.../TestHALeaseManagementModule.java | 75 -
.../TestHBaseOmidClientConfiguration.java | 43 -
.../transaction/TestHBaseTransactionClient.java | 465 -------
.../TestHBaseTransactionManager.java | 102 --
.../yahoo/omid/transaction/TestMultiplePut.java | 134 --
.../yahoo/omid/transaction/TestReadPath.java | 105 --
.../yahoo/omid/transaction/TestShadowCells.java | 508 -------
.../transaction/TestSingleColumnFamily.java | 108 --
.../yahoo/omid/transaction/TestTSOModule.java | 98 --
.../omid/transaction/TestTTableBehaviour.java | 155 ---
.../transaction/TestTransactionCleanup.java | 119 --
.../transaction/TestTransactionConflict.java | 308 -----
.../omid/transaction/TestTxMgrFailover.java | 353 -----
.../yahoo/omid/transaction/TestUpdateScan.java | 218 ---
.../apache/omid/transaction/OmidTestBase.java | 257 ++++
.../TestAsynchronousPostCommitter.java | 323 +++++
.../apache/omid/transaction/TestAutoFlush.java | 62 +
.../omid/transaction/TestBasicTransaction.java | 440 ++++++
.../apache/omid/transaction/TestCellUtils.java | 230 ++++
.../omid/transaction/TestColumnIterator.java | 96 ++
.../apache/omid/transaction/TestDeletion.java | 299 ++++
.../TestEndToEndScenariosWithHA.java | 412 ++++++
.../apache/omid/transaction/TestFilters.java | 191 +++
.../TestHALeaseManagementModule.java | 75 +
.../TestHBaseOmidClientConfiguration.java | 43 +
.../transaction/TestHBaseTransactionClient.java | 465 +++++++
.../TestHBaseTransactionManager.java | 102 ++
.../omid/transaction/TestMultiplePut.java | 134 ++
.../apache/omid/transaction/TestReadPath.java | 105 ++
.../omid/transaction/TestShadowCells.java | 508 +++++++
.../transaction/TestSingleColumnFamily.java | 108 ++
.../apache/omid/transaction/TestTSOModule.java | 98 ++
.../omid/transaction/TestTTableBehaviour.java | 155 +++
.../transaction/TestTransactionCleanup.java | 119 ++
.../transaction/TestTransactionConflict.java | 308 +++++
.../omid/transaction/TestTxMgrFailover.java | 353 +++++
.../apache/omid/transaction/TestUpdateScan.java | 218 +++
.../src/test/resources/log4j.properties | 13 +-
.../resources/test-hbase-omid-client-config.yml | 18 +-
hbase-commit-table/pom.xml | 12 +-
.../DefaultHBaseCommitTableStorageModule.java | 89 --
.../committable/hbase/HBaseCommitTable.java | 422 ------
.../hbase/HBaseCommitTableConfig.java | 82 --
.../hbase/HBaseCommitTableStorageModule.java | 37 -
.../DefaultHBaseCommitTableStorageModule.java | 89 ++
.../committable/hbase/HBaseCommitTable.java | 422 ++++++
.../hbase/HBaseCommitTableConfig.java | 82 ++
.../hbase/HBaseCommitTableStorageModule.java | 37 +
.../committable/hbase/TestHBaseCommitTable.java | 298 ----
.../committable/hbase/TestHBaseCommitTable.java | 298 ++++
hbase-common/pom.xml | 6 +-
.../java/com/yahoo/omid/HBaseConfigModule.java | 83 --
.../omid/committable/hbase/KeyGenerator.java | 29 -
.../hbase/KeyGeneratorImplementations.java | 160 ---
.../omid/committable/hbase/RegionSplitter.java | 259 ----
.../com/yahoo/omid/tools/hbase/HBaseLogin.java | 38 -
.../omid/tools/hbase/SecureHBaseConfig.java | 55 -
.../com/yahoo/omid/transaction/CellInfo.java | 59 -
.../com/yahoo/omid/transaction/CellUtils.java | 406 ------
.../java/org/apache/omid/HBaseConfigModule.java | 83 ++
.../omid/committable/hbase/KeyGenerator.java | 29 +
.../hbase/KeyGeneratorImplementations.java | 160 +++
.../omid/committable/hbase/RegionSplitter.java | 258 ++++
.../org/apache/omid/tools/hbase/HBaseLogin.java | 38 +
.../omid/tools/hbase/SecureHBaseConfig.java | 55 +
.../org/apache/omid/transaction/CellInfo.java | 59 +
.../org/apache/omid/transaction/CellUtils.java | 406 ++++++
.../hbase/TestHBaseCommitTableKey.java | 53 -
.../hbase/TestHBaseCommitTableKey.java | 53 +
hbase-coprocessor/pom.xml | 16 +-
.../yahoo/omid/transaction/CompactorUtil.java | 107 --
.../transaction/HRegionCellGetterAdapter.java | 39 -
.../yahoo/omid/transaction/OmidCompactor.java | 140 --
.../hbase/regionserver/CompactorScanner.java | 14 +-
.../apache/omid/transaction/CompactorUtil.java | 107 ++
.../transaction/HRegionCellGetterAdapter.java | 39 +
.../apache/omid/transaction/OmidCompactor.java | 140 ++
.../TSOForHBaseCompactorTestModule.java | 120 --
.../yahoo/omid/transaction/TestCompaction.java | 1299 ------------------
.../omid/transaction/TestCompactorScanner.java | 106 --
.../TSOForHBaseCompactorTestModule.java | 120 ++
.../apache/omid/transaction/TestCompaction.java | 1299 ++++++++++++++++++
.../omid/transaction/TestCompactorScanner.java | 106 ++
hbase-coprocessor/src/test/resources/log4j.xml | 4 +-
hbase-shims/hbase-0/pom.xml | 2 +-
.../main/java/com/yahoo/omid/HBaseShims.java | 59 -
.../main/java/org/apache/omid/HBaseShims.java | 59 +
hbase-shims/hbase-1/pom.xml | 2 +-
.../main/java/com/yahoo/omid/HBaseShims.java | 58 -
.../main/java/org/apache/omid/HBaseShims.java | 58 +
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 10 +-
.../omid/tools/hbase/OmidTableManager.java | 203 ---
.../omid/tools/hbase/OmidTableManager.java | 203 +++
.../omid/tools/hbase/TestOmidTableManager.java | 119 --
.../omid/tools/hbase/TestOmidTableManager.java | 119 ++
metrics/pom.xml | 2 +-
.../omid/metrics/AbstractMetricsConfig.java | 43 -
.../java/com/yahoo/omid/metrics/Counter.java | 46 -
.../main/java/com/yahoo/omid/metrics/Gauge.java | 28 -
.../java/com/yahoo/omid/metrics/Histogram.java | 36 -
.../main/java/com/yahoo/omid/metrics/Meter.java | 34 -
.../java/com/yahoo/omid/metrics/Metric.java | 25 -
.../com/yahoo/omid/metrics/MetricsProvider.java | 41 -
.../com/yahoo/omid/metrics/MetricsRegistry.java | 61 -
.../yahoo/omid/metrics/MetricsRegistryMap.java | 105 --
.../com/yahoo/omid/metrics/MetricsUtils.java | 47 -
.../yahoo/omid/metrics/NullMetricsProvider.java | 135 --
.../main/java/com/yahoo/omid/metrics/Timer.java | 28 -
.../omid/metrics/AbstractMetricsConfig.java | 43 +
.../java/org/apache/omid/metrics/Counter.java | 46 +
.../java/org/apache/omid/metrics/Gauge.java | 28 +
.../java/org/apache/omid/metrics/Histogram.java | 36 +
.../java/org/apache/omid/metrics/Meter.java | 34 +
.../java/org/apache/omid/metrics/Metric.java | 25 +
.../apache/omid/metrics/MetricsProvider.java | 41 +
.../apache/omid/metrics/MetricsRegistry.java | 61 +
.../apache/omid/metrics/MetricsRegistryMap.java | 105 ++
.../org/apache/omid/metrics/MetricsUtils.java | 47 +
.../omid/metrics/NullMetricsProvider.java | 135 ++
.../java/org/apache/omid/metrics/Timer.java | 28 +
pom.xml | 25 +-
statemachine/pom.xml | 2 +-
.../com/yahoo/statemachine/StateMachine.java | 256 ----
.../statemachine/StateMachineLogParser.java | 163 ---
.../org/apache/statemachine/StateMachine.java | 256 ++++
.../statemachine/StateMachineLogParser.java | 163 +++
.../yahoo/statemachine/TestStateMachine.java | 100 --
.../apache/statemachine/TestStateMachine.java | 100 ++
timestamp-storage/pom.xml | 10 +-
.../DefaultHBaseTimestampStorageModule.java | 81 --
.../DefaultZKTimestampStorageModule.java | 57 -
.../storage/HBaseTimestampStorage.java | 89 --
.../storage/HBaseTimestampStorageConfig.java | 60 -
.../storage/HBaseTimestampStorageModule.java | 36 -
.../timestamp/storage/TimestampStorage.java | 28 -
.../yahoo/omid/timestamp/storage/ZKModule.java | 49 -
.../timestamp/storage/ZKTimestampPaths.java | 28 -
.../timestamp/storage/ZKTimestampStorage.java | 90 --
.../storage/ZKTimestampStorageModule.java | 37 -
.../DefaultHBaseTimestampStorageModule.java | 81 ++
.../DefaultZKTimestampStorageModule.java | 57 +
.../storage/HBaseTimestampStorage.java | 89 ++
.../storage/HBaseTimestampStorageConfig.java | 60 +
.../storage/HBaseTimestampStorageModule.java | 36 +
.../timestamp/storage/TimestampStorage.java | 28 +
.../apache/omid/timestamp/storage/ZKModule.java | 49 +
.../timestamp/storage/ZKTimestampPaths.java | 28 +
.../timestamp/storage/ZKTimestampStorage.java | 90 ++
.../storage/ZKTimestampStorageModule.java | 37 +
.../storage/TestHBaseTimestampStorage.java | 136 --
.../storage/TestZKTimestampStorage.java | 241 ----
.../storage/TestHBaseTimestampStorage.java | 136 ++
.../storage/TestZKTimestampStorage.java | 241 ++++
.../src/test/resources/log4j.properties | 15 +-
transaction-client/pom.xml | 12 +-
.../omid/transaction/AbstractTransaction.java | 217 ---
.../transaction/AbstractTransactionManager.java | 502 -------
.../transaction/CommitTimestampLocator.java | 38 -
.../omid/transaction/PostCommitActions.java | 41 -
.../omid/transaction/RollbackException.java | 35 -
.../com/yahoo/omid/transaction/Transaction.java | 76 -
.../omid/transaction/TransactionException.java | 31 -
.../omid/transaction/TransactionManager.java | 66 -
.../TransactionManagerException.java | 32 -
.../yahoo/omid/tso/client/AbortException.java | 27 -
.../java/com/yahoo/omid/tso/client/CellId.java | 24 -
.../yahoo/omid/tso/client/ClosingException.java | 28 -
.../omid/tso/client/ConnectionException.java | 30 -
.../omid/tso/client/ForwardingTSOFuture.java | 64 -
.../tso/client/HandshakeFailedException.java | 28 -
.../yahoo/omid/tso/client/MockTSOClient.java | 102 --
.../yahoo/omid/tso/client/NewTSOException.java | 27 -
.../tso/client/OmidClientConfiguration.java | 177 ---
.../tso/client/ServiceUnavailableException.java | 32 -
.../com/yahoo/omid/tso/client/TSOClient.java | 933 -------------
.../com/yahoo/omid/tso/client/TSOFuture.java | 25 -
.../com/yahoo/omid/tso/client/TSOProtocol.java | 56 -
.../yahoo/omid/tso/util/DummyCellIdImpl.java | 35 -
.../omid/transaction/AbstractTransaction.java | 217 +++
.../transaction/AbstractTransactionManager.java | 502 +++++++
.../transaction/CommitTimestampLocator.java | 38 +
.../omid/transaction/PostCommitActions.java | 41 +
.../omid/transaction/RollbackException.java | 35 +
.../apache/omid/transaction/Transaction.java | 76 +
.../omid/transaction/TransactionException.java | 31 +
.../omid/transaction/TransactionManager.java | 66 +
.../TransactionManagerException.java | 32 +
.../apache/omid/tso/client/AbortException.java | 27 +
.../java/org/apache/omid/tso/client/CellId.java | 24 +
.../omid/tso/client/ClosingException.java | 28 +
.../omid/tso/client/ConnectionException.java | 30 +
.../omid/tso/client/ForwardingTSOFuture.java | 64 +
.../tso/client/HandshakeFailedException.java | 28 +
.../apache/omid/tso/client/MockTSOClient.java | 102 ++
.../apache/omid/tso/client/NewTSOException.java | 27 +
.../tso/client/OmidClientConfiguration.java | 177 +++
.../tso/client/ServiceUnavailableException.java | 32 +
.../org/apache/omid/tso/client/TSOClient.java | 933 +++++++++++++
.../org/apache/omid/tso/client/TSOFuture.java | 25 +
.../org/apache/omid/tso/client/TSOProtocol.java | 56 +
.../apache/omid/tso/util/DummyCellIdImpl.java | 35 +
.../src/main/resources/omid-client-config.yml | 6 +-
.../omid/tso/client/TestMockTSOClient.java | 72 -
.../tso/client/TestOmidClientConfiguration.java | 32 -
.../omid/tso/client/TestMockTSOClient.java | 72 +
.../tso/client/TestOmidClientConfiguration.java | 32 +
tso-server/bin/omid.sh | 6 +-
tso-server/maven/assembly/bin.xml | 6 +-
tso-server/pom.xml | 20 +-
.../src/main/java/com/yahoo/omid/tso/Cache.java | 32 -
.../com/yahoo/omid/tso/CacheEvaluation.java | 105 --
.../java/com/yahoo/omid/tso/CommitHashMap.java | 81 --
.../com/yahoo/omid/tso/DisruptorModule.java | 36 -
.../yahoo/omid/tso/FatalExceptionHandler.java | 51 -
.../yahoo/omid/tso/HALeaseManagementModule.java | 81 --
.../tso/InMemoryCommitTableStorageModule.java | 35 -
.../tso/InMemoryTimestampStorageModule.java | 35 -
.../com/yahoo/omid/tso/LeaseManagement.java | 52 -
.../java/com/yahoo/omid/tso/LeaseManager.java | 310 -----
.../main/java/com/yahoo/omid/tso/LongCache.java | 85 --
.../java/com/yahoo/omid/tso/MockPanicker.java | 30 -
.../com/yahoo/omid/tso/MonitoringContext.java | 77 --
.../yahoo/omid/tso/NetworkInterfaceUtils.java | 126 --
.../main/java/com/yahoo/omid/tso/Panicker.java | 22 -
.../yahoo/omid/tso/PausableTimestampOracle.java | 70 -
.../yahoo/omid/tso/PersistenceProcessor.java | 30 -
.../omid/tso/PersistenceProcessorImpl.java | 431 ------
.../java/com/yahoo/omid/tso/ReplyProcessor.java | 43 -
.../com/yahoo/omid/tso/ReplyProcessorImpl.java | 236 ----
.../com/yahoo/omid/tso/RequestProcessor.java | 31 -
.../yahoo/omid/tso/RequestProcessorImpl.java | 309 -----
.../java/com/yahoo/omid/tso/RetryProcessor.java | 24 -
.../com/yahoo/omid/tso/RetryProcessorImpl.java | 185 ---
.../com/yahoo/omid/tso/SystemExitPanicker.java | 32 -
.../com/yahoo/omid/tso/TSOChannelHandler.java | 285 ----
.../main/java/com/yahoo/omid/tso/TSOModule.java | 67 -
.../main/java/com/yahoo/omid/tso/TSOServer.java | 131 --
.../com/yahoo/omid/tso/TSOServerConfig.java | 176 ---
.../com/yahoo/omid/tso/TSOStateManager.java | 95 --
.../com/yahoo/omid/tso/TSOStateManagerImpl.java | 78 --
.../com/yahoo/omid/tso/TimestampOracle.java | 34 -
.../com/yahoo/omid/tso/TimestampOracleImpl.java | 169 ---
.../com/yahoo/omid/tso/TsoServerDaemon.java | 60 -
.../omid/tso/VoidLeaseManagementModule.java | 92 --
.../com/yahoo/omid/tso/VoidLeaseManager.java | 53 -
.../main/java/org/apache/omid/tso/Cache.java | 32 +
.../org/apache/omid/tso/CacheEvaluation.java | 105 ++
.../java/org/apache/omid/tso/CommitHashMap.java | 81 ++
.../org/apache/omid/tso/DisruptorModule.java | 36 +
.../apache/omid/tso/FatalExceptionHandler.java | 51 +
.../omid/tso/HALeaseManagementModule.java | 81 ++
.../tso/InMemoryCommitTableStorageModule.java | 35 +
.../tso/InMemoryTimestampStorageModule.java | 35 +
.../org/apache/omid/tso/LeaseManagement.java | 52 +
.../java/org/apache/omid/tso/LeaseManager.java | 310 +++++
.../java/org/apache/omid/tso/LongCache.java | 85 ++
.../java/org/apache/omid/tso/MockPanicker.java | 30 +
.../org/apache/omid/tso/MonitoringContext.java | 77 ++
.../apache/omid/tso/NetworkInterfaceUtils.java | 126 ++
.../main/java/org/apache/omid/tso/Panicker.java | 22 +
.../omid/tso/PausableTimestampOracle.java | 70 +
.../apache/omid/tso/PersistenceProcessor.java | 30 +
.../omid/tso/PersistenceProcessorImpl.java | 431 ++++++
.../org/apache/omid/tso/ReplyProcessor.java | 43 +
.../org/apache/omid/tso/ReplyProcessorImpl.java | 236 ++++
.../org/apache/omid/tso/RequestProcessor.java | 31 +
.../apache/omid/tso/RequestProcessorImpl.java | 309 +++++
.../org/apache/omid/tso/RetryProcessor.java | 24 +
.../org/apache/omid/tso/RetryProcessorImpl.java | 185 +++
.../org/apache/omid/tso/SystemExitPanicker.java | 32 +
.../org/apache/omid/tso/TSOChannelHandler.java | 285 ++++
.../java/org/apache/omid/tso/TSOModule.java | 67 +
.../java/org/apache/omid/tso/TSOServer.java | 131 ++
.../org/apache/omid/tso/TSOServerConfig.java | 176 +++
.../org/apache/omid/tso/TSOStateManager.java | 95 ++
.../apache/omid/tso/TSOStateManagerImpl.java | 78 ++
.../org/apache/omid/tso/TimestampOracle.java | 34 +
.../apache/omid/tso/TimestampOracleImpl.java | 169 +++
.../org/apache/omid/tso/TsoServerDaemon.java | 60 +
.../omid/tso/VoidLeaseManagementModule.java | 92 ++
.../org/apache/omid/tso/VoidLeaseManager.java | 53 +
tso-server/src/main/resources/default-omid.yml | 74 +-
tso-server/src/main/resources/log4j.properties | 13 +-
.../src/test/java/com/yahoo/omid/TestUtils.java | 114 --
.../yahoo/omid/tso/PausableLeaseManager.java | 117 --
.../yahoo/omid/tso/ProgrammableTSOServer.java | 287 ----
.../java/com/yahoo/omid/tso/TSOMockModule.java | 76 -
.../com/yahoo/omid/tso/TSOServerConfigTest.java | 29 -
.../test/java/com/yahoo/omid/tso/TestBatch.java | 158 ---
.../com/yahoo/omid/tso/TestLeaseManager.java | 418 ------
.../java/com/yahoo/omid/tso/TestLongCache.java | 122 --
.../java/com/yahoo/omid/tso/TestPanicker.java | 163 ---
.../omid/tso/TestPersistenceProcessor.java | 221 ---
.../yahoo/omid/tso/TestRequestProcessor.java | 186 ---
.../com/yahoo/omid/tso/TestRetryProcessor.java | 126 --
.../omid/tso/TestTSOChannelHandlerNetty.java | 321 -----
.../com/yahoo/omid/tso/TestTSOStateManager.java | 121 --
.../com/yahoo/omid/tso/TestTimestampOracle.java | 117 --
.../omid/tso/client/TSOClientAccessor.java | 29 -
.../yahoo/omid/tso/client/TSOClientOneShot.java | 63 -
.../com/yahoo/omid/tso/client/TSOClientRaw.java | 150 --
...tionOfTSOClientServerBasicFunctionality.java | 233 ----
.../client/TestTSOClientConnectionToTSO.java | 284 ----
...stTSOClientRequestAndResponseBehaviours.java | 423 ------
.../client/TestTSOClientResponseHandling.java | 111 --
.../tso/client/TestUnconnectedTSOClient.java | 83 --
.../test/java/org/apache/omid/TestUtils.java | 114 ++
.../apache/omid/tso/PausableLeaseManager.java | 117 ++
.../apache/omid/tso/ProgrammableTSOServer.java | 287 ++++
.../java/org/apache/omid/tso/TSOMockModule.java | 76 +
.../apache/omid/tso/TSOServerConfigTest.java | 29 +
.../java/org/apache/omid/tso/TestBatch.java | 158 +++
.../org/apache/omid/tso/TestLeaseManager.java | 418 ++++++
.../java/org/apache/omid/tso/TestLongCache.java | 122 ++
.../java/org/apache/omid/tso/TestPanicker.java | 163 +++
.../omid/tso/TestPersistenceProcessor.java | 221 +++
.../apache/omid/tso/TestRequestProcessor.java | 186 +++
.../org/apache/omid/tso/TestRetryProcessor.java | 126 ++
.../omid/tso/TestTSOChannelHandlerNetty.java | 321 +++++
.../apache/omid/tso/TestTSOStateManager.java | 121 ++
.../apache/omid/tso/TestTimestampOracle.java | 117 ++
.../omid/tso/client/TSOClientAccessor.java | 29 +
.../omid/tso/client/TSOClientOneShot.java | 63 +
.../apache/omid/tso/client/TSOClientRaw.java | 150 ++
...tionOfTSOClientServerBasicFunctionality.java | 233 ++++
.../client/TestTSOClientConnectionToTSO.java | 284 ++++
...stTSOClientRequestAndResponseBehaviours.java | 423 ++++++
.../client/TestTSOClientResponseHandling.java | 111 ++
.../tso/client/TestUnconnectedTSOClient.java | 83 ++
tso-server/src/test/resources/log4j.properties | 15 +-
tso-server/src/test/resources/test-omid.yml | 8 +-
417 files changed, 26656 insertions(+), 26685 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index dead464..03517e6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,4 +17,4 @@ lib/
# TestNG
*/test-output/
-common/src/main/java/com/yahoo/omid/proto/
+common/src/main/java/org/apache/omid/proto/
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/bin/benchmarks.sh
----------------------------------------------------------------------
diff --git a/benchmarks/bin/benchmarks.sh b/benchmarks/bin/benchmarks.sh
index b7a037e..55b2031 100755
--- a/benchmarks/bin/benchmarks.sh
+++ b/benchmarks/bin/benchmarks.sh
@@ -20,7 +20,7 @@ for j in ../lib/*.jar; do
done
tso() {
- exec java $JVM_FLAGS -Dlog4j.configuration=log4j.xml -cp $CLASSPATH com.yahoo.omid.benchmarks.tso.TSOServerBenchmark $@
+ exec java $JVM_FLAGS -Dlog4j.configuration=log4j.xml -cp $CLASSPATH org.apache.omid.benchmarks.tso.TSOServerBenchmark $@
}
usage() {
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index 9991a3d..8ec06db 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -16,22 +16,22 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>transaction-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>metrics</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>codahale-metrics</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
deleted file mode 100644
index cd690e2..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.benchmarks.hbase;
-
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParametersDelegate;
-import com.codahale.metrics.ConsoleReporter;
-import com.codahale.metrics.Meter;
-import com.codahale.metrics.MetricFilter;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.Timer;
-import com.codahale.metrics.graphite.Graphite;
-import com.codahale.metrics.graphite.GraphiteReporter;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.committable.hbase.KeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.BadRandomKeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.BucketKeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.FullRandomKeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations.SeqKeyGenerator;
-import com.yahoo.omid.tools.hbase.HBaseLogin;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-import java.net.InetSocketAddress;
-import java.util.concurrent.TimeUnit;
-
-public class HBaseCommitTableTester {
-
- private static class Config {
-
- @Parameter(names = "-fullRandomAlgo", description = "Full random algo")
- boolean fullRandomAlgo = false;
-
- @Parameter(names = "-badRandomAlgo", description = "The original algo")
- boolean badRandomAlgo = false;
-
- @Parameter(names = "-bucketAlgo", description = "Bucketing algorithm")
- boolean bucketingAlgo = false;
-
- @Parameter(names = "-seqAlgo", description = "Sequential algorithm")
- boolean seqAlgo = false;
-
- @Parameter(names = "-batchSize", description = "batch size")
- int batchSize = 10000;
-
- @Parameter(names = "-graphite", description = "graphite server to report to")
- String graphite = null;
-
- @ParametersDelegate
- SecureHBaseConfig loginFlags = new SecureHBaseConfig();
- }
-
- @SuppressWarnings("InfiniteLoopStatement")
- public static void main(String[] args) throws Exception {
- Config config = new Config();
- new JCommander(config, args);
-
- Configuration hbaseConfig = HBaseConfiguration.create();
-
- final KeyGenerator keygen;
- if (config.fullRandomAlgo) {
- keygen = new FullRandomKeyGenerator();
- } else if (config.badRandomAlgo) {
- keygen = new BadRandomKeyGenerator();
- } else if (config.bucketingAlgo) {
- keygen = new BucketKeyGenerator();
- } else if (config.seqAlgo) {
- keygen = new SeqKeyGenerator();
- } else {
- throw new IllegalArgumentException("Not supported keygen type");
- }
-
- HBaseLogin.loginIfNeeded(config.loginFlags);
-
- HBaseCommitTableConfig commitTableConfig = new HBaseCommitTableConfig();
- CommitTable commitTable = new HBaseCommitTable(hbaseConfig, commitTableConfig, keygen);
-
- CommitTable.Writer writer = commitTable.getWriter();
-
- MetricRegistry metrics = new MetricRegistry();
- if (config.graphite != null) {
- String parts[] = config.graphite.split(":");
- String host = parts[0];
- Integer port = Integer.valueOf(parts[1]);
-
- final Graphite graphite = new Graphite(new InetSocketAddress(host, port));
- final GraphiteReporter reporter = GraphiteReporter.forRegistry(metrics)
- .prefixedWith("omid-hbase." + keygen.getClass().getSimpleName())
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .filter(MetricFilter.ALL)
- .build(graphite);
- reporter.start(10, TimeUnit.SECONDS);
- }
- final ConsoleReporter reporter = ConsoleReporter.forRegistry(metrics)
- .convertRatesTo(TimeUnit.SECONDS)
- .convertDurationsTo(TimeUnit.MILLISECONDS)
- .build();
- reporter.start(10, TimeUnit.SECONDS);
-
- Timer flushTimer = metrics.timer("flush");
- Meter commitsMeter = metrics.meter("commits");
-
- int i = 0;
- long ts = 0;
- while (true) {
- writer.addCommittedTransaction(ts++, ts++);
- if (i++ == config.batchSize) {
- commitsMeter.mark(i);
- long start = System.nanoTime();
- writer.flush();
- flushTimer.update((System.nanoTime() - start), TimeUnit.NANOSECONDS);
- i = 0;
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
deleted file mode 100644
index eaef6e7..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.benchmarks.tso;
-
-import com.google.common.util.concurrent.RateLimiter;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.inject.Binder;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.yahoo.omid.benchmarks.utils.IntegerGenerator;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.Counter;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.Timer;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import com.yahoo.omid.tso.client.AbortException;
-import com.yahoo.omid.tso.client.CellId;
-import com.yahoo.omid.tso.client.OmidClientConfiguration;
-import com.yahoo.omid.tso.client.TSOClient;
-import com.yahoo.omid.tso.client.TSOFuture;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static com.codahale.metrics.MetricRegistry.name;
-
-class RawTxRunner implements Runnable {
-
- private static final Logger LOG = LoggerFactory.getLogger(RawTxRunner.class);
-
- private static volatile int txRunnerCounter = 0;
- private int txRunnerId = txRunnerCounter++;
-
- // Config params
- private final int writesetSize;
- private final boolean fixedWriteSetSize;
- private final long commitDelayInMs;
- private final int percentageOfReadOnlyTxs;
- private final IntegerGenerator cellIdGenerator;
- private final Random randomGen;
-
- // Main elements
- private final TSOClient tsoClient;
- private final CommitTable.Client commitTableClient;
-
- // Asynchronous executor for tx post begin sequence: TimestampListener -> Committer -> CommitListener
- private final ScheduledExecutorService callbackExec =
- Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
- .setNameFormat("tx-runner-" + txRunnerId + "-callback")
- .setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- LOG.error("Thread {} threw exception", t, e);
- }
- }).build());
-
- // Statistics to save
- private final Timer timestampTimer;
- private final Timer commitTimer;
- private final Timer abortTimer;
- private final Counter errorCounter;
-
- // Allows to setup a maximum rate for the client in req/sec
- private final RateLimiter rateLimiter;
-
- // Is this TxRunner still running?
- private volatile boolean isRunning = false;
-
- RawTxRunner(final TSOServerBenchmarkConfig expConfig) throws IOException, InterruptedException {
-
- // Injector configuration
- List<Module> guiceModules = new ArrayList<>();
- guiceModules.add(new Module() {
- @Override
- public void configure(Binder binder) {
- binder.bind(MetricsRegistry.class).toInstance(expConfig.getMetrics());
- }
- });
- guiceModules.add(expConfig.getCommitTableStoreModule());
- Injector injector = Guice.createInjector(guiceModules);
-
- // Tx Runner config
- this.writesetSize = expConfig.getWritesetSize();
- this.fixedWriteSetSize = expConfig.isFixedWritesetSize();
- this.commitDelayInMs = expConfig.getCommitDelayInMs();
- this.percentageOfReadOnlyTxs = expConfig.getPercentageOfReadOnlyTxs();
- this.cellIdGenerator = expConfig.getCellIdGenerator();
- this.randomGen = new Random(System.currentTimeMillis() * txRunnerId); // to make it channel dependent
-
- int txRateInReqPerSec = expConfig.getTxRateInRequestPerSecond();
- long warmUpPeriodInSecs = expConfig.getWarmUpPeriodInSecs();
-
- LOG.info("TxRunner-{} [ Tx Rate (Req per Sec) -> {} ]", txRunnerId, txRateInReqPerSec);
- LOG.info("TxRunner-{} [ Warm Up Period -> {} Secs ]", txRunnerId, warmUpPeriodInSecs);
- LOG.info("TxRunner-{} [ Cell Id Distribution Generator -> {} ]", txRunnerId, expConfig.getCellIdGenerator().getClass());
- LOG.info("TxRunner-{} [ Max Tx Size -> {} Fixed: {} ]", txRunnerId, writesetSize, fixedWriteSetSize);
- LOG.info("TxRunner-{} [ Commit delay -> {} Ms ]", txRunnerId, commitDelayInMs);
- LOG.info("TxRunner-{} [ % of Read-Only Tx -> {} % ]", txRunnerId, percentageOfReadOnlyTxs);
-
- // Commit table client initialization
- CommitTable commitTable = injector.getInstance(CommitTable.class);
- this.commitTableClient = commitTable.getClient();
-
- // Stat initialization
- MetricsRegistry metrics = injector.getInstance(MetricsRegistry.class);
- String hostName = InetAddress.getLocalHost().getHostName();
- this.timestampTimer = metrics.timer(name("tx_runner", Integer.toString(txRunnerId), hostName, "timestamp"));
- this.commitTimer = metrics.timer(name("tx_runner", Integer.toString(txRunnerId), hostName, "commit"));
- this.abortTimer = metrics.timer(name("tx_runner", Integer.toString(txRunnerId), hostName, "abort"));
- this.errorCounter = metrics.counter(name("tx_runner", Integer.toString(txRunnerId), hostName, "errors"));
- LOG.info("TxRunner-{} [ Metrics provider module -> {} ]", txRunnerId, expConfig.getMetrics().getClass());
-
- // TSO Client initialization
- OmidClientConfiguration tsoClientConf = expConfig.getOmidClientConfiguration();
- this.tsoClient = TSOClient.newInstance(tsoClientConf);
- LOG.info("TxRunner-{} [ Connection Type {}/Connection String {} ]", txRunnerId,
- tsoClientConf.getConnectionType(), tsoClientConf.getConnectionString());
-
- // Limiter for configured request per second
- this.rateLimiter = RateLimiter.create((double) txRateInReqPerSec, warmUpPeriodInSecs, TimeUnit.SECONDS);
- }
-
- @Override
- public void run() {
-
- isRunning = true;
-
- while (isRunning) {
- rateLimiter.acquire();
- long tsRequestTime = System.nanoTime();
- final TSOFuture<Long> tsFuture = tsoClient.getNewStartTimestamp();
- tsFuture.addListener(new TimestampListener(tsFuture, tsRequestTime), callbackExec);
- }
-
- shutdown();
-
- }
-
- public void stop() {
- isRunning = false;
- }
-
- private void shutdown() {
-
- try {
- LOG.info("Finishing TxRunner in 3 secs", txRunnerId);
- boolean wasSuccess = callbackExec.awaitTermination(3, TimeUnit.SECONDS);
- if (!wasSuccess) {
- callbackExec.shutdownNow();
- }
- commitTableClient.close();
- tsoClient.close().get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- // ignore
- } catch (ExecutionException | IOException e) {
- // ignore
- } finally {
- LOG.info("TxRunner {} finished", txRunnerId);
- }
-
- }
-
- private class TimestampListener implements Runnable {
-
- final TSOFuture<Long> tsFuture;
- final long tsRequestTime;
-
- TimestampListener(TSOFuture<Long> tsFuture, long tsRequestTime) {
- this.tsFuture = tsFuture;
- this.tsRequestTime = tsRequestTime;
- }
-
- @Override
- public void run() {
-
- try {
- long txId = tsFuture.get();
- timestampTimer.update(System.nanoTime() - tsRequestTime);
- if (commitDelayInMs <= 0) {
- callbackExec.execute(new Committer(txId));
- } else {
- callbackExec.schedule(new Committer(txId), commitDelayInMs, TimeUnit.MILLISECONDS);
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- errorCounter.inc();
- } catch (ExecutionException e) {
- errorCounter.inc();
- }
-
- }
-
- }
-
- private class Committer implements Runnable {
-
- final long txId;
-
- Committer(long txId) {
- this.txId = txId;
- }
-
- @Override
- public void run() {
-
- int txWritesetSize = calculateTxWritesetSize();
-
- if (txWritesetSize == 0) {
- return; // Read only tx, no need to commit
- }
- // Otherwise, we create the writeset...
- final Set<CellId> cells = new HashSet<>();
- for (byte i = 0; i < txWritesetSize; i++) {
- long cellId = cellIdGenerator.nextInt();
- cells.add(new DummyCellIdImpl(cellId));
- }
- // ... and we commit the transaction
- long startCommitTimeInNs = System.nanoTime();
- final TSOFuture<Long> commitFuture = tsoClient.commit(txId, cells);
- commitFuture.addListener(new CommitListener(txId, commitFuture, startCommitTimeInNs), callbackExec);
-
- }
-
- private int calculateTxWritesetSize() {
- int txSize = 0;
- boolean readOnly = (randomGen.nextFloat() * 100) < percentageOfReadOnlyTxs;
- if (!readOnly) {
- if (fixedWriteSetSize) {
- txSize = writesetSize;
- } else {
- txSize = randomGen.nextInt(writesetSize) + 1;
- }
- }
- return txSize;
- }
-
- }
-
- private class CommitListener implements Runnable {
-
- final long txId;
- final long commitRequestTime;
- final TSOFuture<Long> commitFuture;
-
- CommitListener(long txId, TSOFuture<Long> commitFuture, long commitRequestTime) {
- this.txId = txId;
- this.commitFuture = commitFuture;
- this.commitRequestTime = commitRequestTime;
- }
-
- @Override
- public void run() {
-
- try {
- commitFuture.get();
- commitTableClient.completeTransaction(txId).get();
- commitTimer.update(System.nanoTime() - commitRequestTime);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- errorCounter.inc();
- } catch (ExecutionException e) {
- if (e.getCause() instanceof AbortException) {
- abortTimer.update(System.nanoTime() - commitRequestTime);
- } else {
- errorCounter.inc();
- }
- }
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
deleted file mode 100644
index b39d28a..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.benchmarks.tso;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Benchmark using directly TSOClient to connect to the TSO Server
- */
-public class TSOServerBenchmark implements Closeable {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOServerBenchmark.class);
-
- private volatile boolean isCleaningDone = false;
-
- private final TSOServerBenchmarkConfig expConfig;
-
- // Clients triggering txs (threads) & corresponding executor
- private final ArrayList<RawTxRunner> txRunners = new ArrayList<>();
- private final ScheduledExecutorService txRunnerExec;
-
- private TSOServerBenchmark(TSOServerBenchmarkConfig expConfig) throws IOException {
-
- this.expConfig = expConfig;
-
- // Executor for TxRunners (Clients triggering transactions)
- Thread.UncaughtExceptionHandler uncaughtExceptionHandler = new Thread.UncaughtExceptionHandler() {
- @Override
- public void uncaughtException(Thread t, Throwable e) {
- LOG.error("Thread {} threw exception", t, e);
- }
- };
- ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder()
- .setNameFormat("tx-runner-%d")
- .setUncaughtExceptionHandler(uncaughtExceptionHandler);
- this.txRunnerExec = Executors.newScheduledThreadPool(expConfig.getTxRunners(), threadFactoryBuilder.build());
-
- }
-
- public static void main(String[] args) throws Exception {
-
- final TSOServerBenchmarkConfig config = new TSOServerBenchmarkConfig();
-
- final int nOfTxRunners = config.getTxRunners();
-
- final long benchmarkRunLengthInMins = config.getBenchmarkRunLengthInMins();
-
- try (TSOServerBenchmark tsoBenchmark = new TSOServerBenchmark(config)) {
-
- tsoBenchmark.attachShutDownHook();
-
- LOG.info("----- Starting TSO Benchmark [ {} TxRunner clients ] -----", nOfTxRunners);
-
- for (int i = 0; i < nOfTxRunners; ++i) {
- tsoBenchmark.createTxRunner();
- }
-
- LOG.info("Benchmark run lenght {} Mins", benchmarkRunLengthInMins);
- TimeUnit.MINUTES.sleep(benchmarkRunLengthInMins);
-
- } finally {
- LOG.info("----- TSO Benchmark complete - Check metrics from individual clients in log -----");
- }
-
- }
-
- private void attachShutDownHook() {
- Runtime.getRuntime().addShutdownHook(new Thread("benchmark-cleaner") {
- @Override
- public void run() {
- if (!isCleaningDone) {
- close();
- }
- }
- });
- LOG.info("Shutdown Hook Attached");
- }
-
- private void createTxRunner() throws IOException, InterruptedException, ExecutionException {
-
- RawTxRunner txRunner = new RawTxRunner(expConfig);
- txRunnerExec.submit(txRunner);
-
- txRunners.add(txRunner);
-
- }
-
- @Override
- public void close() {
-
- // Stop clients
- for (RawTxRunner txRunner : txRunners) {
- txRunner.stop();
- }
-
- // Shutdown executor
- try {
- LOG.info("Closing TxRunner Executor in 10 secs");
- boolean wasSuccess = txRunnerExec.awaitTermination(10, TimeUnit.SECONDS);
- if (!wasSuccess) {
- txRunnerExec.shutdownNow();
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.info("Interrupted whilst shutting down TxRunner Executor!");
- } finally {
- LOG.info("TxRunner Executor stopped");
- }
-
- isCleaningDone = true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
deleted file mode 100644
index 150040c..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.benchmarks.tso;
-
-import com.google.inject.AbstractModule;
-import com.yahoo.omid.YAMLUtils;
-import com.yahoo.omid.benchmarks.utils.IntegerGenerator;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import com.yahoo.omid.tso.client.OmidClientConfiguration;
-
-public class TSOServerBenchmarkConfig extends SecureHBaseConfig {
-
- private static final String DEFAULT_CONFIG_FILE_NAME = "tso-server-benchmark-config.yml";
- private static final String CONFIG_FILE_NAME = "default-tso-server-benchmark-config.yml";
-
- private long benchmarkRunLengthInMins;
-
- private int txRunners;
- private int txRateInRequestPerSecond;
- private long warmUpPeriodInSecs;
- private IntegerGenerator cellIdGenerator;
- private int writesetSize;
- private boolean fixedWritesetSize;
- private int percentageOfReadOnlyTxs;
- private long commitDelayInMs;
-
- private OmidClientConfiguration omidClientConfiguration;
- private AbstractModule commitTableStoreModule;
-
- private MetricsRegistry metrics;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Instantiation
- // ----------------------------------------------------------------------------------------------------------------
-
- TSOServerBenchmarkConfig() {
- this(CONFIG_FILE_NAME);
- }
-
- TSOServerBenchmarkConfig(String configFileName) {
- new YAMLUtils().loadSettings(DEFAULT_CONFIG_FILE_NAME, configFileName, this);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters and setters for config params
- // ----------------------------------------------------------------------------------------------------------------
-
- public long getBenchmarkRunLengthInMins() {
- return benchmarkRunLengthInMins;
- }
-
- public void setBenchmarkRunLengthInMins(long benchmarkRunLengthInMins) {
- this.benchmarkRunLengthInMins = benchmarkRunLengthInMins;
- }
-
- public int getTxRunners() {
- return txRunners;
- }
-
- public void setTxRunners(int txRunners) {
- this.txRunners = txRunners;
- }
-
- public int getTxRateInRequestPerSecond() {
- return txRateInRequestPerSecond;
- }
-
- public void setTxRateInRequestPerSecond(int txRateInRequestPerSecond) {
- this.txRateInRequestPerSecond = txRateInRequestPerSecond;
- }
-
- public long getWarmUpPeriodInSecs() {
- return warmUpPeriodInSecs;
- }
-
- public void setWarmUpPeriodInSecs(long warmUpPeriodInSecs) {
- this.warmUpPeriodInSecs = warmUpPeriodInSecs;
- }
-
- public IntegerGenerator getCellIdGenerator() {
- return cellIdGenerator;
- }
-
- public void setCellIdGenerator(IntegerGenerator cellIdGenerator) {
- this.cellIdGenerator = cellIdGenerator;
- }
-
- public int getWritesetSize() {
- return writesetSize;
- }
-
- public void setWritesetSize(int writesetSize) {
- this.writesetSize = writesetSize;
- }
-
- public boolean isFixedWritesetSize() {
- return fixedWritesetSize;
- }
-
- public void setFixedWritesetSize(boolean fixedWritesetSize) {
- this.fixedWritesetSize = fixedWritesetSize;
- }
-
- public int getPercentageOfReadOnlyTxs() {
- return percentageOfReadOnlyTxs;
- }
-
- public void setPercentageOfReadOnlyTxs(int percentageOfReadOnlyTxs) {
- this.percentageOfReadOnlyTxs = percentageOfReadOnlyTxs;
- }
-
- public long getCommitDelayInMs() {
- return commitDelayInMs;
- }
-
- public void setCommitDelayInMs(long commitDelayInMs) {
- this.commitDelayInMs = commitDelayInMs;
- }
-
- public OmidClientConfiguration getOmidClientConfiguration() {
- return omidClientConfiguration;
- }
-
- public void setOmidClientConfiguration(OmidClientConfiguration omidClientConfiguration) {
- this.omidClientConfiguration = omidClientConfiguration;
- }
-
- public AbstractModule getCommitTableStoreModule() {
- return commitTableStoreModule;
- }
-
- public void setCommitTableStoreModule(AbstractModule commitTableStoreModule) {
- this.commitTableStoreModule = commitTableStoreModule;
- }
-
- public MetricsRegistry getMetrics() {
- return metrics;
- }
-
- public void setMetrics(MetricsRegistry metrics) {
- this.metrics = metrics;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/Generator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/Generator.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/Generator.java
deleted file mode 100644
index bbd106f..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/Generator.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you
- * may not use this file except in compliance with the License. You
- * may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License. See accompanying
- * LICENSE file.
- */
-
-package com.yahoo.omid.benchmarks.utils;
-
-/**
- * An expression that generates a sequence of string values, following some distribution (Uniform, Zipfian, Sequential, etc.)
- */
-public abstract class Generator {
- /**
- * Generate the next string in the distribution.
- */
- public abstract String nextString();
-
- /**
- * Return the previous string generated by the distribution; e.g., returned from the last nextString() call.
- * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
- * been called, lastString() should return something reasonable.
- */
- public abstract String lastString();
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/IntegerGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/IntegerGenerator.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/IntegerGenerator.java
deleted file mode 100644
index 9f73e71..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/IntegerGenerator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you
- * may not use this file except in compliance with the License. You
- * may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License. See accompanying
- * LICENSE file.
- */
-
-package com.yahoo.omid.benchmarks.utils;
-
-/**
- * A generator that is capable of generating ints as well as strings
- *
- * @author cooperb
- *
- */
-public abstract class IntegerGenerator extends Generator {
- int lastint;
-
- /**
- * Set the last value generated. IntegerGenerator subclasses must use this call
- * to properly set the last string value, or the lastString() and lastInt() calls won't work.
- */
- protected void setLastInt(int last) {
- lastint = last;
- }
-
- /**
- * Return the next value as an int. When overriding this method, be sure to call setLastString() properly, or the lastString() call won't work.
- */
- public abstract int nextInt();
-
- /**
- * Generate the next string in the distribution.
- */
- public String nextString() {
- return "" + nextInt();
- }
-
- /**
- * Return the previous string generated by the distribution; e.g., returned from the last nextString() call.
- * Calling lastString() should not advance the distribution or have any side effects. If nextString() has not yet
- * been called, lastString() should return something reasonable.
- */
- @Override
- public String lastString() {
- return "" + lastInt();
- }
-
- /**
- * Return the previous int generated by the distribution. This call is unique to IntegerGenerator subclasses, and assumes
- * IntegerGenerator subclasses always return ints for nextInt() (e.g. not arbitrary strings).
- */
- public int lastInt() {
- return lastint;
- }
-
- /**
- * Return the expected value (mean) of the values this generator will return.
- */
- public abstract double mean();
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ScrambledZipfianGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ScrambledZipfianGenerator.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ScrambledZipfianGenerator.java
deleted file mode 100644
index 10c4103..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/ScrambledZipfianGenerator.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you
- * may not use this file except in compliance with the License. You
- * may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- * implied. See the License for the specific language governing
- * permissions and limitations under the License. See accompanying
- * LICENSE file.
- */
-
-package com.yahoo.omid.benchmarks.utils;
-
-
-/**
- * A generator of a zipfian distribution. It produces a sequence of items, such that some items are more popular than others, according
- * to a zipfian distribution. When you construct an instance of this class, you specify the number of items in the set to draw from, either
- * by specifying an itemcount (so that the sequence is of items from 0 to itemcount-1) or by specifying a min and a max (so that the sequence is of
- * items from min to max inclusive). After you construct the instance, you can change the number of items by calling nextInt(itemcount) or nextLong(itemcount).
- *
- * Unlike @ZipfianGenerator, this class scatters the "popular" items across the itemspace. Use this, instead of @ZipfianGenerator, if you
- * don't want the head of the distribution (the popular items) clustered together.
- */
-public class ScrambledZipfianGenerator extends IntegerGenerator {
- public static final double ZETAN = 26.46902820178302;
- public static final double USED_ZIPFIAN_CONSTANT = 0.99;
- public static final long ITEM_COUNT = 10000000000L;
-
- ZipfianGenerator gen;
- long _min, _max, _itemcount;
-
- /******************************* Constructors **************************************/
-
- /**
- * Create a zipfian generator for the specified number of items.
- * @param _items The number of items in the distribution.
- */
- public ScrambledZipfianGenerator(long _items) {
- this(0, _items - 1);
- }
-
- /**
- * Create a zipfian generator for items between min and max.
- * @param _min The smallest integer to generate in the sequence.
- * @param _max The largest integer to generate in the sequence.
- */
- public ScrambledZipfianGenerator(long _min, long _max) {
- this(_min, _max, ZipfianGenerator.ZIPFIAN_CONSTANT);
- }
-
- /**
- * Create a zipfian generator for the specified number of items using the specified zipfian constant.
- *
- * @param _items The number of items in the distribution.
- * @param _zipfianconstant The zipfian constant to use.
- */
- /*
-// not supported, as the value of zeta depends on the zipfian constant, and we have only precomputed zeta for one zipfian constant
- public ScrambledZipfianGenerator(long _items, double _zipfianconstant)
- {
- this(0,_items-1,_zipfianconstant);
- }
-*/
-
- /**
- * Create a zipfian generator for items between min and max (inclusive) for the specified zipfian constant. If you
- * use a zipfian constant other than 0.99, this will take a long time to complete because we need to recompute zeta.
- * @param min The smallest integer to generate in the sequence.
- * @param max The largest integer to generate in the sequence.
- * @param _zipfianconstant The zipfian constant to use.
- */
- public ScrambledZipfianGenerator(long min, long max, double _zipfianconstant) {
- _min = min;
- _max = max;
- _itemcount = _max - _min + 1;
- if (_zipfianconstant == USED_ZIPFIAN_CONSTANT) {
- gen = new ZipfianGenerator(0, ITEM_COUNT, _zipfianconstant, ZETAN);
- } else {
- gen = new ZipfianGenerator(0, ITEM_COUNT, _zipfianconstant);
- }
- }
-
- /**************************************************************************************************/
-
- /**
- * Return the next int in the sequence.
- */
- @Override
- public int nextInt() {
- return (int) nextLong();
- }
-
- /**
- * Return the next long in the sequence.
- */
- public long nextLong() {
- long ret = gen.nextLong();
- ret = _min + FNVhash64(ret) % _itemcount;
- setLastInt((int) ret);
- return ret;
- }
-
- public static void main(String[] args) {
- double newzetan = ZipfianGenerator.zetastatic(ITEM_COUNT, ZipfianGenerator.ZIPFIAN_CONSTANT);
- System.out.println("zetan: " + newzetan);
- System.exit(0);
-
- ScrambledZipfianGenerator gen = new ScrambledZipfianGenerator(10000);
-
- for (int i = 0; i < 1000000; i++) {
- System.out.println("" + gen.nextInt());
- }
- }
-
- /**
- * since the values are scrambled (hopefully uniformly), the mean is simply the middle of the range.
- */
- @Override
- public double mean() {
- return ((double) (((long) _min) + (long) _max)) / 2.0;
- }
-
- /**
- * 64 bit FNV hash. Produces more "random" hashes than (say) String.hashCode().
- *
- * @param val The value to hash.
- * @return The hash value
- */
- public static long FNVhash64(long val) {
- //from http://en.wikipedia.org/wiki/Fowler_Noll_Vo_hash
- long hashval = FNV_offset_basis_64;
-
- for (int i = 0; i < 8; i++) {
- long octet = val & 0x00ff;
- val = val >> 8;
-
- hashval = hashval ^ octet;
- hashval = hashval * FNV_prime_64;
- //hashval = hashval ^ octet;
- }
- return Math.abs(hashval);
- }
-
- public static final long FNV_offset_basis_64 = 0xCBF29CE484222325L;
- public static final long FNV_prime_64 = 1099511628211L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
deleted file mode 100644
index eb27bd3..0000000
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.benchmarks.utils;
-
-import java.util.Random;
-
-public class UniformGenerator extends IntegerGenerator {
-
- private final Random ramdom = new Random(System.nanoTime());
-
- @Override
- public int nextInt() {
- return ramdom.nextInt(Integer.MAX_VALUE);
- }
-
- @Override
- public double mean() {
- return 0;
- }
-
-}
[27/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
----------------------------------------------------------------------
diff --git a/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java b/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
deleted file mode 100644
index a1ff834..0000000
--- a/statemachine/src/main/java/com/yahoo/statemachine/StateMachine.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.statemachine;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Future;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-public class StateMachine {
-
- private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class);
-
- private static final String HANDLER_METHOD_NAME = "handleEvent";
-
- private static ConcurrentHashMap<Class<?>, ConcurrentHashMap<Class<?>, Method>> stateCaches;
-
- static {
- stateCaches = new ConcurrentHashMap<>();
- }
-
- public static abstract class State {
-
- protected final Fsm fsm;
- private final ConcurrentHashMap<Class<?>, Method> handlerCache;
-
- public State(Fsm fsm) {
- this.fsm = fsm;
-
- ConcurrentHashMap<Class<?>, Method> handlerCache = stateCaches.get(getClass());
- if (handlerCache == null) {
- handlerCache = new ConcurrentHashMap<>();
- ConcurrentHashMap<Class<?>, Method> old = stateCaches.putIfAbsent(getClass(), handlerCache);
- if (old != null) {
- handlerCache = old;
- }
- }
- this.handlerCache = handlerCache;
- }
-
- private Method findHandlerInternal(Class<?> state, Class<?> e) throws NoSuchMethodException {
- Method[] methods = state.getMethods();
- List<Method> candidates = new ArrayList<>();
- for (Method m : methods) {
- if (m.getName().equals(HANDLER_METHOD_NAME)
- && State.class.isAssignableFrom(m.getReturnType())
- && m.getGenericParameterTypes().length == 1) {
- candidates.add(m);
- }
- }
-
- Method best = null;
- for (Method m : candidates) {
- if (m.getParameterTypes()[0].isAssignableFrom(e)) {
- if (best == null) {
- best = m;
- } else if (best.getParameterTypes()[0]
- .isAssignableFrom(m.getParameterTypes()[0])) {
- best = m;
- }
- }
- }
- if (best != null) {
- best.setAccessible(true);
- return best;
- }
- throw new NoSuchMethodException("Handler doesn't exist");
- }
-
- private Method findHandler(Class<?> event) throws NoSuchMethodException {
- Method m = handlerCache.get(event);
- if (m == null) {
- m = findHandlerInternal(getClass(), event);
- Method m2 = handlerCache.putIfAbsent(event, m);
- if (m2 != null) {
- m = m2;
- }
- }
- return m;
- }
-
- State dispatch(Event e) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {
- return (State) findHandler(e.getClass()).invoke(this, e);
- }
-
- }
-
- public interface Event {
- }
-
- public interface Fsm {
- Fsm newChildFsm();
-
- void setInitState(State initState);
-
- void sendEvent(Event e);
-
- Future<?> sendEvent(Event e, long delay, TimeUnit unit);
-
- void deferEvent(DeferrableEvent e);
- }
-
- public static class FsmImpl implements Fsm {
- ScheduledExecutorService executor;
- private State state;
- private Queue<DeferrableEvent> deferred;
-
- public FsmImpl(ScheduledExecutorService executor) {
- this.executor = executor;
- state = null;
- deferred = new ArrayDeque<>();
- }
-
- private void errorDeferredEvents(Throwable t) {
- Queue<DeferrableEvent> oldDeferred = deferred;
- deferred = new ArrayDeque<>();
-
- for (DeferrableEvent e : oldDeferred) {
- e.error(new IllegalStateException(t));
- }
- }
-
- @Override
- public Fsm newChildFsm() {
- return new FsmImpl(executor);
- }
-
- @Override
- public void setInitState(State initState) {
- assert (state == null);
- state = initState;
- }
-
- public State getState() {
- return state;
- }
-
- void setState(final State curState, final State newState) {
- if (curState != state) {
- LOG.error("FSM-{}: Tried to transition from {} to {}, but current state is {}",
- getFsmId(), state, newState, curState);
- throw new IllegalArgumentException();
- }
- state = newState;
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("FSM-{}: State transition {} -> {}", getFsmId(), curState, newState);
- }
- }
-
- boolean processEvent(Event e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("FSM-{}: Received event {}@{} in state {}@{}",
- getFsmId(), e.getClass().getSimpleName(),
- System.identityHashCode(e),
- state.getClass().getSimpleName(),
- System.identityHashCode(state));
- }
- try {
- State newState = state.dispatch(e);
-
- if (newState != state) {
- setState(state, newState);
- return true;
- }
- } catch (Throwable t) {
- LOG.error("Caught throwable while handling event", t);
- errorDeferredEvents(t);
- }
- return false;
- }
-
- class FSMRunnable implements Runnable {
- final Event e;
-
- FSMRunnable(Event e) {
- this.e = e;
- }
-
- @Override
- public void run() {
- boolean stateChanged = processEvent(e);
- while (stateChanged) {
- stateChanged = false;
- Queue<DeferrableEvent> prevDeferred = deferred;
- deferred = new ArrayDeque<>();
- for (DeferrableEvent d : prevDeferred) {
- if (stateChanged) {
- deferred.add(d);
- } else if (processEvent(d)) {
- stateChanged = true;
- }
- }
- }
- }
- }
-
- @Override
- public void sendEvent(final Event e) {
- executor.submit(new FSMRunnable(e));
- }
-
- @Override
- public Future<?> sendEvent(final Event e, final long delay, final TimeUnit unit) {
- return executor.schedule(new FSMRunnable(e), delay, unit);
- }
-
- @Override
- public void deferEvent(DeferrableEvent e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("FSM-{}: deferred {}@{}",
- getFsmId(), e.getClass().getSimpleName(), System.identityHashCode(e));
- }
- deferred.add(e);
- }
-
- int getFsmId() {
- return System.identityHashCode(this);
- }
-
- @Override
- public void finalize() throws Throwable {
- super.finalize();
- LOG.debug("FSM-{}: Finalizing", getFsmId());
- }
- }
-
- public interface DeferrableEvent extends Event {
- void error(Throwable exception);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
----------------------------------------------------------------------
diff --git a/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java b/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
deleted file mode 100644
index 7b66d8a..0000000
--- a/statemachine/src/main/java/com/yahoo/statemachine/StateMachineLogParser.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.statemachine;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class StateMachineLogParser {
-
- static Pattern fsmPattern = Pattern.compile("FSM-(\\d+):");
-
- static Map<String, List<String>> getFsmEventMap(BufferedReader f) throws IOException {
- String s = f.readLine();
- Map<String, List<String>> map = new HashMap<String, List<String>>();
- while (s != null) {
- Matcher m = fsmPattern.matcher(s);
- if (m.find()) {
- String key = m.group(1);
- if (!map.containsKey(key)) {
- map.put(key, new ArrayList<String>());
- }
- map.get(key).add(s);
- }
- s = f.readLine();
- }
- return map;
- }
-
- static class Tuple {
- final String state1;
- final String state2;
- final String event;
-
- Tuple(String state1, String event, String state2) {
- this.state1 = state1;
- this.state2 = state2;
- this.event = event;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o instanceof Tuple) {
- Tuple t = (Tuple) o;
- return state1.equals(t.state1)
- && state2.equals(t.state2)
- && event.equals(t.event);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return (state1 + event + state2).hashCode();
- }
-
- @Override
- public String toString() {
- return state1 + " + " + event + " = " + state2;
- }
- }
-
- static Pattern hashcodePattern = Pattern.compile("(.+)@(.+)");
- static Pattern subclassPattern = Pattern.compile("(.+)\\$(.+)");
-
- static String cleanupState(String state) {
- Matcher m = hashcodePattern.matcher(state);
- if (m.find()) {
- state = m.group(1);
- }
- m = subclassPattern.matcher(state);
- if (m.find()) {
- state = m.group(2);
- }
- return state;
- }
-
- static Pattern eventPattern = Pattern.compile("Received event (.+)@\\d+ in state (.+)@\\d+");
- static Pattern transitionPattern = Pattern.compile("State transition (.+) -> (.+)");
-
- static Set<Tuple> getStateTuples(Map<String, List<String>> fsms) {
- Set<Tuple> tuples = new HashSet<Tuple>();
- for (List<String> transitions : fsms.values()) {
- String currentEvent = null;
- for (String s : transitions) {
- //System.err.println(s);
- Matcher m = eventPattern.matcher(s);
- if (m.find()) {
- currentEvent = m.group(1);
- continue;
- }
- m = transitionPattern.matcher(s);
- if (m.find()) {
- if (currentEvent == null) {
- System.err.println("event is null");
- }
- String state1 = m.group(1);
- String state2 = m.group(2);
- tuples.add(new Tuple(cleanupState(state1),
- currentEvent,
- cleanupState(state2)));
- continue;
- }
- if (s.contains("deferred")) {
- currentEvent = currentEvent + "[deferred]";
- }
- }
- }
- return tuples;
- }
-
- static void drawDotGraph(Set<Tuple> tuples) {
- Set<String> states = new HashSet<String>();
- for (Tuple t : tuples) {
- states.add(t.state1);
- states.add(t.state2);
- }
-
- System.out.println("digraph finite_state_machine {");
- for (String s : states) {
- System.out.println("\tnode [ shape = circle ] " + s + ";");
- }
- for (Tuple t : tuples) {
- System.out.println("\t" + t.state1 + " -> " + t.state2 + " [ label = \"" + t.event + "\" ];");
- }
- System.out.println("}");
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length == 0) {
- System.out.println("Usage: StateMachineLogParser <logfile> | dot -Tsvg ");
- System.exit(-1);
- }
- BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(args[0])));
- Map<String, List<String>> fsms = getFsmEventMap(f);
- Set<Tuple> tuples = getStateTuples(fsms);
- drawDotGraph(tuples);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/src/main/java/org/apache/statemachine/StateMachine.java
----------------------------------------------------------------------
diff --git a/statemachine/src/main/java/org/apache/statemachine/StateMachine.java b/statemachine/src/main/java/org/apache/statemachine/StateMachine.java
new file mode 100644
index 0000000..184539d
--- /dev/null
+++ b/statemachine/src/main/java/org/apache/statemachine/StateMachine.java
@@ -0,0 +1,256 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.statemachine;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class StateMachine {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StateMachine.class);
+
+ private static final String HANDLER_METHOD_NAME = "handleEvent";
+
+ private static ConcurrentHashMap<Class<?>, ConcurrentHashMap<Class<?>, Method>> stateCaches;
+
+ static {
+ stateCaches = new ConcurrentHashMap<>();
+ }
+
+ public static abstract class State {
+
+ protected final Fsm fsm;
+ private final ConcurrentHashMap<Class<?>, Method> handlerCache;
+
+ public State(Fsm fsm) {
+ this.fsm = fsm;
+
+ ConcurrentHashMap<Class<?>, Method> handlerCache = stateCaches.get(getClass());
+ if (handlerCache == null) {
+ handlerCache = new ConcurrentHashMap<>();
+ ConcurrentHashMap<Class<?>, Method> old = stateCaches.putIfAbsent(getClass(), handlerCache);
+ if (old != null) {
+ handlerCache = old;
+ }
+ }
+ this.handlerCache = handlerCache;
+ }
+
+ private Method findHandlerInternal(Class<?> state, Class<?> e) throws NoSuchMethodException {
+ Method[] methods = state.getMethods();
+ List<Method> candidates = new ArrayList<>();
+ for (Method m : methods) {
+ if (m.getName().equals(HANDLER_METHOD_NAME)
+ && State.class.isAssignableFrom(m.getReturnType())
+ && m.getGenericParameterTypes().length == 1) {
+ candidates.add(m);
+ }
+ }
+
+ Method best = null;
+ for (Method m : candidates) {
+ if (m.getParameterTypes()[0].isAssignableFrom(e)) {
+ if (best == null) {
+ best = m;
+ } else if (best.getParameterTypes()[0]
+ .isAssignableFrom(m.getParameterTypes()[0])) {
+ best = m;
+ }
+ }
+ }
+ if (best != null) {
+ best.setAccessible(true);
+ return best;
+ }
+ throw new NoSuchMethodException("Handler doesn't exist");
+ }
+
+ private Method findHandler(Class<?> event) throws NoSuchMethodException {
+ Method m = handlerCache.get(event);
+ if (m == null) {
+ m = findHandlerInternal(getClass(), event);
+ Method m2 = handlerCache.putIfAbsent(event, m);
+ if (m2 != null) {
+ m = m2;
+ }
+ }
+ return m;
+ }
+
+ State dispatch(Event e) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {
+ return (State) findHandler(e.getClass()).invoke(this, e);
+ }
+
+ }
+
+ public interface Event {
+ }
+
+ public interface Fsm {
+ Fsm newChildFsm();
+
+ void setInitState(State initState);
+
+ void sendEvent(Event e);
+
+ Future<?> sendEvent(Event e, long delay, TimeUnit unit);
+
+ void deferEvent(DeferrableEvent e);
+ }
+
+ public static class FsmImpl implements Fsm {
+ ScheduledExecutorService executor;
+ private State state;
+ private Queue<DeferrableEvent> deferred;
+
+ public FsmImpl(ScheduledExecutorService executor) {
+ this.executor = executor;
+ state = null;
+ deferred = new ArrayDeque<>();
+ }
+
+ private void errorDeferredEvents(Throwable t) {
+ Queue<DeferrableEvent> oldDeferred = deferred;
+ deferred = new ArrayDeque<>();
+
+ for (DeferrableEvent e : oldDeferred) {
+ e.error(new IllegalStateException(t));
+ }
+ }
+
+ @Override
+ public Fsm newChildFsm() {
+ return new FsmImpl(executor);
+ }
+
+ @Override
+ public void setInitState(State initState) {
+ assert (state == null);
+ state = initState;
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ void setState(final State curState, final State newState) {
+ if (curState != state) {
+ LOG.error("FSM-{}: Tried to transition from {} to {}, but current state is {}",
+ getFsmId(), state, newState, curState);
+ throw new IllegalArgumentException();
+ }
+ state = newState;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FSM-{}: State transition {} -> {}", getFsmId(), curState, newState);
+ }
+ }
+
+ boolean processEvent(Event e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FSM-{}: Received event {}@{} in state {}@{}",
+ getFsmId(), e.getClass().getSimpleName(),
+ System.identityHashCode(e),
+ state.getClass().getSimpleName(),
+ System.identityHashCode(state));
+ }
+ try {
+ State newState = state.dispatch(e);
+
+ if (newState != state) {
+ setState(state, newState);
+ return true;
+ }
+ } catch (Throwable t) {
+ LOG.error("Caught throwable while handling event", t);
+ errorDeferredEvents(t);
+ }
+ return false;
+ }
+
+ class FSMRunnable implements Runnable {
+ final Event e;
+
+ FSMRunnable(Event e) {
+ this.e = e;
+ }
+
+ @Override
+ public void run() {
+ boolean stateChanged = processEvent(e);
+ while (stateChanged) {
+ stateChanged = false;
+ Queue<DeferrableEvent> prevDeferred = deferred;
+ deferred = new ArrayDeque<>();
+ for (DeferrableEvent d : prevDeferred) {
+ if (stateChanged) {
+ deferred.add(d);
+ } else if (processEvent(d)) {
+ stateChanged = true;
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void sendEvent(final Event e) {
+ executor.submit(new FSMRunnable(e));
+ }
+
+ @Override
+ public Future<?> sendEvent(final Event e, final long delay, final TimeUnit unit) {
+ return executor.schedule(new FSMRunnable(e), delay, unit);
+ }
+
+ @Override
+ public void deferEvent(DeferrableEvent e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FSM-{}: deferred {}@{}",
+ getFsmId(), e.getClass().getSimpleName(), System.identityHashCode(e));
+ }
+ deferred.add(e);
+ }
+
+ int getFsmId() {
+ return System.identityHashCode(this);
+ }
+
+ @Override
+ public void finalize() throws Throwable {
+ super.finalize();
+ LOG.debug("FSM-{}: Finalizing", getFsmId());
+ }
+ }
+
+ public interface DeferrableEvent extends Event {
+ void error(Throwable exception);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/src/main/java/org/apache/statemachine/StateMachineLogParser.java
----------------------------------------------------------------------
diff --git a/statemachine/src/main/java/org/apache/statemachine/StateMachineLogParser.java b/statemachine/src/main/java/org/apache/statemachine/StateMachineLogParser.java
new file mode 100644
index 0000000..15e5fe0
--- /dev/null
+++ b/statemachine/src/main/java/org/apache/statemachine/StateMachineLogParser.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.statemachine;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class StateMachineLogParser {
+
+ static Pattern fsmPattern = Pattern.compile("FSM-(\\d+):");
+
+ static Map<String, List<String>> getFsmEventMap(BufferedReader f) throws IOException {
+ String s = f.readLine();
+ Map<String, List<String>> map = new HashMap<String, List<String>>();
+ while (s != null) {
+ Matcher m = fsmPattern.matcher(s);
+ if (m.find()) {
+ String key = m.group(1);
+ if (!map.containsKey(key)) {
+ map.put(key, new ArrayList<String>());
+ }
+ map.get(key).add(s);
+ }
+ s = f.readLine();
+ }
+ return map;
+ }
+
+ static class Tuple {
+ final String state1;
+ final String state2;
+ final String event;
+
+ Tuple(String state1, String event, String state2) {
+ this.state1 = state1;
+ this.state2 = state2;
+ this.event = event;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Tuple) {
+ Tuple t = (Tuple) o;
+ return state1.equals(t.state1)
+ && state2.equals(t.state2)
+ && event.equals(t.event);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return (state1 + event + state2).hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return state1 + " + " + event + " = " + state2;
+ }
+ }
+
+ static Pattern hashcodePattern = Pattern.compile("(.+)@(.+)");
+ static Pattern subclassPattern = Pattern.compile("(.+)\\$(.+)");
+
+ static String cleanupState(String state) {
+ Matcher m = hashcodePattern.matcher(state);
+ if (m.find()) {
+ state = m.group(1);
+ }
+ m = subclassPattern.matcher(state);
+ if (m.find()) {
+ state = m.group(2);
+ }
+ return state;
+ }
+
+ static Pattern eventPattern = Pattern.compile("Received event (.+)@\\d+ in state (.+)@\\d+");
+ static Pattern transitionPattern = Pattern.compile("State transition (.+) -> (.+)");
+
+ static Set<Tuple> getStateTuples(Map<String, List<String>> fsms) {
+ Set<Tuple> tuples = new HashSet<Tuple>();
+ for (List<String> transitions : fsms.values()) {
+ String currentEvent = null;
+ for (String s : transitions) {
+ //System.err.println(s);
+ Matcher m = eventPattern.matcher(s);
+ if (m.find()) {
+ currentEvent = m.group(1);
+ continue;
+ }
+ m = transitionPattern.matcher(s);
+ if (m.find()) {
+ if (currentEvent == null) {
+ System.err.println("event is null");
+ }
+ String state1 = m.group(1);
+ String state2 = m.group(2);
+ tuples.add(new Tuple(cleanupState(state1),
+ currentEvent,
+ cleanupState(state2)));
+ continue;
+ }
+ if (s.contains("deferred")) {
+ currentEvent = currentEvent + "[deferred]";
+ }
+ }
+ }
+ return tuples;
+ }
+
+ static void drawDotGraph(Set<Tuple> tuples) {
+ Set<String> states = new HashSet<String>();
+ for (Tuple t : tuples) {
+ states.add(t.state1);
+ states.add(t.state2);
+ }
+
+ System.out.println("digraph finite_state_machine {");
+ for (String s : states) {
+ System.out.println("\tnode [ shape = circle ] " + s + ";");
+ }
+ for (Tuple t : tuples) {
+ System.out.println("\t" + t.state1 + " -> " + t.state2 + " [ label = \"" + t.event + "\" ];");
+ }
+ System.out.println("}");
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length == 0) {
+ System.out.println("Usage: StateMachineLogParser <logfile> | dot -Tsvg ");
+ System.exit(-1);
+ }
+ BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(args[0])));
+ Map<String, List<String>> fsms = getFsmEventMap(f);
+ Set<Tuple> tuples = getStateTuples(fsms);
+ drawDotGraph(tuples);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java b/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
deleted file mode 100644
index 9f90163..0000000
--- a/statemachine/src/test/java/com/yahoo/statemachine/TestStateMachine.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.statemachine;
-
-import com.yahoo.statemachine.StateMachine.Fsm;
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-import com.yahoo.statemachine.StateMachine.State;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-
-public class TestStateMachine {
- private final static Logger LOG = LoggerFactory.getLogger(TestStateMachine.class);
-
- static class TestEvent implements StateMachine.DeferrableEvent {
- CountDownLatch latch = new CountDownLatch(1);
- Throwable t = null;
- int i = 0;
-
- public void error(Throwable t) {
- this.t = t;
- latch.countDown();
- }
-
- public void success(int i) {
- this.i = i;
- latch.countDown();
- }
-
- public int get() throws InterruptedException, Throwable {
- latch.await();
- if (t != null) {
- throw t;
- }
- return i;
- }
- }
-
- static class CompletingState extends State {
- int completed = 0;
-
- CompletingState(Fsm fsm) {
- super(fsm);
- }
-
- public State handleEvent(TestEvent e) {
- e.success(completed++);
- return this;
- }
- }
-
- static class DeferringState extends State {
- int count = 0;
-
- DeferringState(Fsm fsm) {
- super(fsm);
- }
-
- public State handleEvent(TestEvent e) {
- if (count++ < 5) {
- fsm.deferEvent(e);
- return this;
- } else {
- fsm.deferEvent(e);
- return new CompletingState(fsm);
- }
- }
- }
-
- @Test(timeOut = 60000)
- public void testOrdering() throws Throwable {
- Fsm fsm = new FsmImpl(Executors.newSingleThreadScheduledExecutor());
- fsm.setInitState(new DeferringState(fsm));
- for (int i = 0; i < 10; i++) {
- fsm.sendEvent(new TestEvent());
- }
- TestEvent te = new TestEvent();
- fsm.sendEvent(te);
- Assert.assertEquals(10, te.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/src/test/java/org/apache/statemachine/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/statemachine/src/test/java/org/apache/statemachine/TestStateMachine.java b/statemachine/src/test/java/org/apache/statemachine/TestStateMachine.java
new file mode 100644
index 0000000..567ec11
--- /dev/null
+++ b/statemachine/src/test/java/org/apache/statemachine/TestStateMachine.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.statemachine;
+
+import org.apache.statemachine.StateMachine.Fsm;
+import org.apache.statemachine.StateMachine.FsmImpl;
+import org.apache.statemachine.StateMachine.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+
+public class TestStateMachine {
+ private final static Logger LOG = LoggerFactory.getLogger(TestStateMachine.class);
+
+ static class TestEvent implements StateMachine.DeferrableEvent {
+ CountDownLatch latch = new CountDownLatch(1);
+ Throwable t = null;
+ int i = 0;
+
+ public void error(Throwable t) {
+ this.t = t;
+ latch.countDown();
+ }
+
+ public void success(int i) {
+ this.i = i;
+ latch.countDown();
+ }
+
+ public int get() throws InterruptedException, Throwable {
+ latch.await();
+ if (t != null) {
+ throw t;
+ }
+ return i;
+ }
+ }
+
+ static class CompletingState extends State {
+ int completed = 0;
+
+ CompletingState(Fsm fsm) {
+ super(fsm);
+ }
+
+ public State handleEvent(TestEvent e) {
+ e.success(completed++);
+ return this;
+ }
+ }
+
+ static class DeferringState extends State {
+ int count = 0;
+
+ DeferringState(Fsm fsm) {
+ super(fsm);
+ }
+
+ public State handleEvent(TestEvent e) {
+ if (count++ < 5) {
+ fsm.deferEvent(e);
+ return this;
+ } else {
+ fsm.deferEvent(e);
+ return new CompletingState(fsm);
+ }
+ }
+ }
+
+ @Test(timeOut = 60000)
+ public void testOrdering() throws Throwable {
+ Fsm fsm = new FsmImpl(Executors.newSingleThreadScheduledExecutor());
+ fsm.setInitState(new DeferringState(fsm));
+ for (int i = 0; i < 10; i++) {
+ fsm.sendEvent(new TestEvent());
+ }
+ TestEvent te = new TestEvent();
+ fsm.sendEvent(te);
+ Assert.assertEquals(10, te.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index 3e8e60e..3d379ad 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -17,12 +17,12 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
@@ -116,7 +116,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -127,7 +127,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
deleted file mode 100644
index 4d84cc8..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.name.Names;
-import com.yahoo.omid.HBaseConfigModule;
-
-import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
-import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
-
-/**
- * This class is instantiated by the yaml parser.
- * Snake_yaml needs a public POJO style class to work properly with all the setters and getters.
- */
-public class DefaultHBaseTimestampStorageModule extends AbstractModule {
-
- private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
- private String familyName = DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
- private String keytab;
- private String principal;
-
- @Override
- public void configure() {
- bindConstant().annotatedWith(Names.named(HBaseTimestampStorageConfig.TIMESTAMP_STORAGE_TABLE_NAME_KEY)).to(tableName);
- bindConstant().annotatedWith(Names.named(HBaseTimestampStorageConfig.TIMESTAMP_STORAGE_CF_NAME_KEY)).to(familyName);
- install(new HBaseConfigModule(principal, keytab));
- install(new HBaseTimestampStorageModule());
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // WARNING: Do not remove getters/setters, needed by snake_yaml!
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getTableName() {
- return tableName;
- }
-
- public void setTableName(String tableName) {
- this.tableName = tableName;
- }
-
- public String getFamilyName() {
- return familyName;
- }
-
- public void setFamilyName(String familyName) {
- this.familyName = familyName;
- }
-
- public String getPrincipal() {
- return principal;
- }
-
- public void setPrincipal(String principal) {
- this.principal = principal;
- }
-
- public String getKeytab() {
- return keytab;
- }
-
- public void setKeytab(String keytab) {
- this.keytab = keytab;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
deleted file mode 100644
index 525a2d7..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import com.google.inject.AbstractModule;
-
-/**
- * This class is instantiated by the yaml parser.
- * Snake_yaml needs a public POJO style class to work properly with all the setters and getters.
- */
-public class DefaultZKTimestampStorageModule extends AbstractModule {
-
- private String zkCluster = "localhost:2181";
- private String namespace = "omid";
-
- @Override
- public void configure() {
- install(new ZKModule(zkCluster, namespace));
- install(new ZKTimestampStorageModule());
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // WARNING: Do not remove getters/setters, needed by snake_yaml!
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getZkCluster() {
- return zkCluster;
- }
-
- public void setZkCluster(String zkCluster) {
- this.zkCluster = zkCluster;
- }
-
- public String getNamespace() {
- return namespace;
- }
-
- public void setNamespace(String namespace) {
- this.namespace = namespace;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
deleted file mode 100644
index f56aea4..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorage.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-
-import static com.google.common.base.Charsets.UTF_8;
-
-/**
- * Stores the max timestamp assigned by the TO in HBase.
- * It's always written non-transactionally in the same row and column
- */
-public class HBaseTimestampStorage implements TimestampStorage {
-
- private static final long INITIAL_MAX_TS_VALUE = 0;
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseTimestampStorage.class);
-
- // ROW and COLUMN to write the assigned timestamp
- private static final byte[] TSO_ROW = "MAX_TIMESTAMP_R".getBytes(UTF_8);
- private static final byte[] TSO_QUALIFIER = "MAX_TIMESTAMP_Q".getBytes(UTF_8);
-
- private final HTable table;
- private final byte[] cfName;
-
- @Inject
- public HBaseTimestampStorage(Configuration hbaseConfig, HBaseTimestampStorageConfig config) throws IOException {
- this.table = new HTable(hbaseConfig, config.getTableName());
- this.cfName = config.getFamilyName().getBytes(UTF_8);
- }
-
- @Override
- public void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException {
- if (newMaxTimestamp < 0) {
- LOG.error("Negative value received for maxTimestamp: {}", newMaxTimestamp);
- throw new IllegalArgumentException("Negative value received for maxTimestamp" + newMaxTimestamp);
- }
- Put put = new Put(TSO_ROW);
- put.add(cfName, TSO_QUALIFIER, Bytes.toBytes(newMaxTimestamp));
- byte[] previousVal = null;
- if (previousMaxTimestamp != INITIAL_MAX_TS_VALUE) {
- previousVal = Bytes.toBytes(previousMaxTimestamp);
- }
- if (!table.checkAndPut(TSO_ROW, cfName, TSO_QUALIFIER, previousVal, put)) {
- throw new IOException("Previous max timestamp is incorrect");
- }
- }
-
- @Override
- public long getMaxTimestamp() throws IOException {
- Get get = new Get(TSO_ROW);
- get.addColumn(cfName, TSO_QUALIFIER);
-
- Result result = table.get(get);
- if (result.containsColumn(cfName, TSO_QUALIFIER)) {
- return Bytes.toLong(result.getValue(cfName, TSO_QUALIFIER));
- } else {
- // This happens for example when a new cluster is created
- return INITIAL_MAX_TS_VALUE;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
deleted file mode 100644
index 829201b..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageConfig.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import com.google.inject.Inject;
-
-import javax.inject.Named;
-
-public class HBaseTimestampStorageConfig {
-
- public static final String TIMESTAMP_STORAGE_TABLE_NAME_KEY = "omid.timestampstorage.tablename";
- public static final String TIMESTAMP_STORAGE_CF_NAME_KEY = "omid.timestampstorage.cfname";
-
- public static final String DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME = "OMID_TIMESTAMP_TABLE";
- public static final String DEFAULT_TIMESTAMP_STORAGE_CF_NAME = "MAX_TIMESTAMP_CF";
-
- // ----------------------------------------------------------------------------------------------------------------
- // Configuration parameters
- // ----------------------------------------------------------------------------------------------------------------
-
- private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
- private String familyName = DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters and setters
- // ----------------------------------------------------------------------------------------------------------------
-
- public String getTableName() {
- return tableName;
- }
-
- @Inject(optional = true)
- public void setTableName(@Named(TIMESTAMP_STORAGE_TABLE_NAME_KEY) String tableName) {
- this.tableName = tableName;
- }
-
- public String getFamilyName() {
- return familyName;
- }
-
- @Inject(optional = true)
- public void setFamilyName(@Named(TIMESTAMP_STORAGE_CF_NAME_KEY) String familyName) {
- this.familyName = familyName;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
deleted file mode 100644
index 1246e43..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/HBaseTimestampStorageModule.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import com.google.inject.AbstractModule;
-import org.apache.hadoop.conf.Configuration;
-
-import javax.inject.Singleton;
-
-public class HBaseTimestampStorageModule extends AbstractModule {
-
- @Override
- public void configure() {
-
- requireBinding(Configuration.class);
- // Timestamp storage creation
- bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
deleted file mode 100644
index 00c92b7..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/TimestampStorage.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import java.io.IOException;
-
-public interface TimestampStorage {
-
- void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException;
-
- long getMaxTimestamp() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
deleted file mode 100644
index 57dfe71..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKModule.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.zk.ZKUtils;
-import org.apache.curator.framework.CuratorFramework;
-
-import javax.inject.Singleton;
-import java.io.IOException;
-
-//TODO:IK: move to common?
-public class ZKModule extends AbstractModule {
-
- private final String zkCluster;
- private final String namespace;
-
- public ZKModule(String zkCluster, String namespace) {
- this.zkCluster = zkCluster;
- this.namespace = namespace;
- }
-
- @Override
- public void configure() {
- }
-
- @Provides
- @Singleton
- CuratorFramework provideInitializedZookeeperClient() throws IOException, InterruptedException {
- return ZKUtils.initZKClient(zkCluster, namespace, 10);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
deleted file mode 100644
index 88d0ea0..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampPaths.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-public class ZKTimestampPaths {
-
- public static final String TIMESTAMP_ZNODE = "/omid/timestamp";
-
- // Avoid instantiation
- private ZKTimestampPaths() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
deleted file mode 100644
index 173b9fa..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorage.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.atomic.AtomicValue;
-import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
-import org.apache.curator.retry.RetryNTimes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-import java.io.IOException;
-
-import static com.yahoo.omid.timestamp.storage.ZKTimestampPaths.TIMESTAMP_ZNODE;
-
-class ZKTimestampStorage implements TimestampStorage {
-
- private static final Logger LOG = LoggerFactory.getLogger(ZKTimestampStorage.class);
-
- static final long INITIAL_MAX_TS_VALUE = 0;
-
- private final DistributedAtomicLong timestamp;
-
- @Inject
- public ZKTimestampStorage(CuratorFramework zkClient) throws Exception {
- LOG.info("ZK Client state {}", zkClient.getState());
- timestamp = new DistributedAtomicLong(zkClient, TIMESTAMP_ZNODE, new RetryNTimes(3, 1000)); // TODO Configure
- // this?
- if (timestamp.initialize(INITIAL_MAX_TS_VALUE)) {
- LOG.info("Timestamp value in ZNode initialized to {}", INITIAL_MAX_TS_VALUE);
- }
- }
-
- @Override
- public void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException {
-
- if (newMaxTimestamp < 0) {
- LOG.error("Negative value received for maxTimestamp: {}", newMaxTimestamp);
- throw new IllegalArgumentException();
- }
- if (newMaxTimestamp <= previousMaxTimestamp) {
- LOG.error("maxTimestamp {} <= previousMaxTimesamp: {}", newMaxTimestamp, previousMaxTimestamp);
- throw new IllegalArgumentException();
- }
- AtomicValue<Long> compareAndSet;
- try {
- compareAndSet = timestamp.compareAndSet(previousMaxTimestamp, newMaxTimestamp);
- } catch (Exception e) {
- throw new IOException("Problem setting timestamp in ZK", e);
- }
- if (!compareAndSet.succeeded()) { // We have to explicitly check for success (See Curator doc)
- throw new IOException("GetAndSet operation for storing timestamp in ZK did not succeed "
- + compareAndSet.preValue() + " " + compareAndSet.postValue());
- }
-
- }
-
- @Override
- public long getMaxTimestamp() throws IOException {
-
- AtomicValue<Long> atomicValue;
- try {
- atomicValue = timestamp.get();
- } catch (Exception e) {
- throw new IOException("Problem getting data from ZK", e);
- }
- if (!atomicValue.succeeded()) { // We have to explicitly check for success (See Curator doc)
- throw new IOException("Get operation to obtain timestamp from ZK did not succeed");
- }
- return atomicValue.postValue();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java b/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
deleted file mode 100644
index 02be0ea..0000000
--- a/timestamp-storage/src/main/java/com/yahoo/omid/timestamp/storage/ZKTimestampStorageModule.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import com.google.inject.AbstractModule;
-import org.apache.curator.framework.CuratorFramework;
-
-import javax.inject.Singleton;
-
-public class ZKTimestampStorageModule extends AbstractModule {
-
-
- @Override
- public void configure() {
-
- requireBinding(CuratorFramework.class);
- // Timestamp storage creation
- bind(TimestampStorage.class).to(ZKTimestampStorage.class).in(Singleton.class);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
new file mode 100644
index 0000000..b78dae6
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultHBaseTimestampStorageModule.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.name.Names;
+import org.apache.omid.HBaseConfigModule;
+
+import static org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
+import static org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+
+/**
+ * This class is instantiated by the yaml parser.
+ * Snake_yaml needs a public POJO style class to work properly with all the setters and getters.
+ */
+public class DefaultHBaseTimestampStorageModule extends AbstractModule {
+
+ private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+ private String familyName = DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
+ private String keytab;
+ private String principal;
+
+ @Override
+ public void configure() {
+ bindConstant().annotatedWith(Names.named(HBaseTimestampStorageConfig.TIMESTAMP_STORAGE_TABLE_NAME_KEY)).to(tableName);
+ bindConstant().annotatedWith(Names.named(HBaseTimestampStorageConfig.TIMESTAMP_STORAGE_CF_NAME_KEY)).to(familyName);
+ install(new HBaseConfigModule(principal, keytab));
+ install(new HBaseTimestampStorageModule());
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // WARNING: Do not remove getters/setters, needed by snake_yaml!
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getFamilyName() {
+ return familyName;
+ }
+
+ public void setFamilyName(String familyName) {
+ this.familyName = familyName;
+ }
+
+ public String getPrincipal() {
+ return principal;
+ }
+
+ public void setPrincipal(String principal) {
+ this.principal = principal;
+ }
+
+ public String getKeytab() {
+ return keytab;
+ }
+
+ public void setKeytab(String keytab) {
+ this.keytab = keytab;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultZKTimestampStorageModule.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
new file mode 100644
index 0000000..cdf874a
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/DefaultZKTimestampStorageModule.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import com.google.inject.AbstractModule;
+
+/**
+ * This class is instantiated by the yaml parser.
+ * Snake_yaml needs a public POJO style class to work properly with all the setters and getters.
+ */
+public class DefaultZKTimestampStorageModule extends AbstractModule {
+
+ private String zkCluster = "localhost:2181";
+ private String namespace = "omid";
+
+ @Override
+ public void configure() {
+ install(new ZKModule(zkCluster, namespace));
+ install(new ZKTimestampStorageModule());
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // WARNING: Do not remove getters/setters, needed by snake_yaml!
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getZkCluster() {
+ return zkCluster;
+ }
+
+ public void setZkCluster(String zkCluster) {
+ this.zkCluster = zkCluster;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorage.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorage.java
new file mode 100644
index 0000000..9acb8ba
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorage.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+
+import static com.google.common.base.Charsets.UTF_8;
+
+/**
+ * Stores the max timestamp assigned by the TO in HBase.
+ * It's always written non-transactionally in the same row and column
+ */
+public class HBaseTimestampStorage implements TimestampStorage {
+
+ private static final long INITIAL_MAX_TS_VALUE = 0;
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseTimestampStorage.class);
+
+ // ROW and COLUMN to write the assigned timestamp
+ private static final byte[] TSO_ROW = "MAX_TIMESTAMP_R".getBytes(UTF_8);
+ private static final byte[] TSO_QUALIFIER = "MAX_TIMESTAMP_Q".getBytes(UTF_8);
+
+ private final HTable table;
+ private final byte[] cfName;
+
+ @Inject
+ public HBaseTimestampStorage(Configuration hbaseConfig, HBaseTimestampStorageConfig config) throws IOException {
+ this.table = new HTable(hbaseConfig, config.getTableName());
+ this.cfName = config.getFamilyName().getBytes(UTF_8);
+ }
+
+ @Override
+ public void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException {
+ if (newMaxTimestamp < 0) {
+ LOG.error("Negative value received for maxTimestamp: {}", newMaxTimestamp);
+ throw new IllegalArgumentException("Negative value received for maxTimestamp" + newMaxTimestamp);
+ }
+ Put put = new Put(TSO_ROW);
+ put.add(cfName, TSO_QUALIFIER, Bytes.toBytes(newMaxTimestamp));
+ byte[] previousVal = null;
+ if (previousMaxTimestamp != INITIAL_MAX_TS_VALUE) {
+ previousVal = Bytes.toBytes(previousMaxTimestamp);
+ }
+ if (!table.checkAndPut(TSO_ROW, cfName, TSO_QUALIFIER, previousVal, put)) {
+ throw new IOException("Previous max timestamp is incorrect");
+ }
+ }
+
+ @Override
+ public long getMaxTimestamp() throws IOException {
+ Get get = new Get(TSO_ROW);
+ get.addColumn(cfName, TSO_QUALIFIER);
+
+ Result result = table.get(get);
+ if (result.containsColumn(cfName, TSO_QUALIFIER)) {
+ return Bytes.toLong(result.getValue(cfName, TSO_QUALIFIER));
+ } else {
+ // This happens for example when a new cluster is created
+ return INITIAL_MAX_TS_VALUE;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageConfig.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageConfig.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageConfig.java
new file mode 100644
index 0000000..45124ef
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageConfig.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import com.google.inject.Inject;
+
+import javax.inject.Named;
+
+public class HBaseTimestampStorageConfig {
+
+ public static final String TIMESTAMP_STORAGE_TABLE_NAME_KEY = "omid.timestampstorage.tablename";
+ public static final String TIMESTAMP_STORAGE_CF_NAME_KEY = "omid.timestampstorage.cfname";
+
+ public static final String DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME = "OMID_TIMESTAMP_TABLE";
+ public static final String DEFAULT_TIMESTAMP_STORAGE_CF_NAME = "MAX_TIMESTAMP_CF";
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Configuration parameters
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private String tableName = DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+ private String familyName = DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters and setters
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ @Inject(optional = true)
+ public void setTableName(@Named(TIMESTAMP_STORAGE_TABLE_NAME_KEY) String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getFamilyName() {
+ return familyName;
+ }
+
+ @Inject(optional = true)
+ public void setFamilyName(@Named(TIMESTAMP_STORAGE_CF_NAME_KEY) String familyName) {
+ this.familyName = familyName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageModule.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageModule.java
new file mode 100644
index 0000000..4cae8b1
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/HBaseTimestampStorageModule.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import com.google.inject.AbstractModule;
+import org.apache.hadoop.conf.Configuration;
+
+import javax.inject.Singleton;
+
+public class HBaseTimestampStorageModule extends AbstractModule {
+
+ @Override
+ public void configure() {
+
+ requireBinding(Configuration.class);
+ // Timestamp storage creation
+ bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/TimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/TimestampStorage.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/TimestampStorage.java
new file mode 100644
index 0000000..f976e83
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/TimestampStorage.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import java.io.IOException;
+
+public interface TimestampStorage {
+
+ void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException;
+
+ long getMaxTimestamp() throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKModule.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKModule.java
new file mode 100644
index 0000000..18ddcd6
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKModule.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import org.apache.omid.zk.ZKUtils;
+import org.apache.curator.framework.CuratorFramework;
+
+import javax.inject.Singleton;
+import java.io.IOException;
+
+//TODO:IK: move to common?
+public class ZKModule extends AbstractModule {
+
+ private final String zkCluster;
+ private final String namespace;
+
+ public ZKModule(String zkCluster, String namespace) {
+ this.zkCluster = zkCluster;
+ this.namespace = namespace;
+ }
+
+ @Override
+ public void configure() {
+ }
+
+ @Provides
+ @Singleton
+ CuratorFramework provideInitializedZookeeperClient() throws IOException, InterruptedException {
+ return ZKUtils.initZKClient(zkCluster, namespace, 10);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampPaths.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampPaths.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampPaths.java
new file mode 100644
index 0000000..7b60a40
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampPaths.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+public class ZKTimestampPaths {
+
+ public static final String TIMESTAMP_ZNODE = "/omid/timestamp";
+
+ // Avoid instantiation
+ private ZKTimestampPaths() {
+ }
+
+}
[45/52] [abbrv] incubator-omid git commit: Merge pull request #90
from yahoo/PR-yahoo-apache
Posted by ik...@apache.org.
Merge pull request #90 from yahoo/PR-yahoo-apache
Move com.yahoo -> org.apache
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/fd8ca18b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/fd8ca18b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/fd8ca18b
Branch: refs/heads/master
Commit: fd8ca18b3bc95883feeb1011c9d8a280265f613c
Parents: 5734cac 9cd856c
Author: Francisco Pérez-Sorrosal <fp...@gmail.com>
Authored: Fri Apr 15 16:10:42 2016 -0700
Committer: Francisco Pérez-Sorrosal <fp...@gmail.com>
Committed: Fri Apr 15 16:10:42 2016 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
benchmarks/bin/benchmarks.sh | 2 +-
benchmarks/pom.xml | 10 +-
.../hbase/HBaseCommitTableTester.java | 137 --
.../yahoo/omid/benchmarks/tso/RawTxRunner.java | 301 ----
.../omid/benchmarks/tso/TSOServerBenchmark.java | 138 --
.../tso/TSOServerBenchmarkConfig.java | 160 ---
.../yahoo/omid/benchmarks/utils/Generator.java | 36 -
.../omid/benchmarks/utils/IntegerGenerator.java | 71 -
.../utils/ScrambledZipfianGenerator.java | 153 ---
.../omid/benchmarks/utils/UniformGenerator.java | 36 -
.../omid/benchmarks/utils/ZipfianGenerator.java | 298 ----
.../hbase/HBaseCommitTableTester.java | 137 ++
.../apache/omid/benchmarks/tso/RawTxRunner.java | 301 ++++
.../omid/benchmarks/tso/TSOServerBenchmark.java | 138 ++
.../tso/TSOServerBenchmarkConfig.java | 160 +++
.../apache/omid/benchmarks/utils/Generator.java | 30 +
.../omid/benchmarks/utils/IntegerGenerator.java | 65 +
.../utils/ScrambledZipfianGenerator.java | 147 ++
.../omid/benchmarks/utils/UniformGenerator.java | 36 +
.../omid/benchmarks/utils/ZipfianGenerator.java | 292 ++++
.../default-tso-server-benchmark-config.yml | 20 +-
codahale-metrics/pom.xml | 4 +-
.../omid/metrics/CodahaleMetricsConfig.java | 100 --
.../omid/metrics/CodahaleMetricsProvider.java | 346 -----
.../omid/metrics/CodahaleMetricsConfig.java | 100 ++
.../omid/metrics/CodahaleMetricsProvider.java | 346 +++++
commit-table/pom.xml | 2 +-
.../com/yahoo/omid/committable/CommitTable.java | 113 --
.../omid/committable/InMemoryCommitTable.java | 143 --
.../yahoo/omid/committable/NullCommitTable.java | 91 --
.../apache/omid/committable/CommitTable.java | 113 ++
.../omid/committable/InMemoryCommitTable.java | 143 ++
.../omid/committable/NullCommitTable.java | 91 ++
.../omid/committable/NullCommitTableTest.java | 70 -
.../omid/committable/NullCommitTableTest.java | 70 +
common/pom.xml | 2 +-
.../src/main/java/com/yahoo/omid/YAMLUtils.java | 81 --
.../main/java/com/yahoo/omid/zk/ZKUtils.java | 57 -
.../main/java/org/apache/omid/YAMLUtils.java | 81 ++
.../main/java/org/apache/omid/zk/ZKUtils.java | 57 +
common/src/main/proto/TSOProto.proto | 2 +-
.../test/java/com/yahoo/omid/YAMLUtilsTest.java | 50 -
.../java/org/apache/omid/YAMLUtilsTest.java | 50 +
examples/pom.xml | 10 +-
examples/run.sh | 6 +-
.../com/yahoo/omid/examples/BasicExample.java | 112 --
.../omid/examples/ConfigurationExample.java | 131 --
.../omid/examples/SnapshotIsolationExample.java | 174 ---
.../org/apache/omid/examples/BasicExample.java | 112 ++
.../omid/examples/ConfigurationExample.java | 131 ++
.../omid/examples/SnapshotIsolationExample.java | 174 +++
.../main/resources/hbase-omid-client-config.yml | 10 +-
examples/src/main/resources/log4j.properties | 3 +-
hbase-client/pom.xml | 18 +-
.../yahoo/omid/transaction/ColumnWrapper.java | 64 -
.../transaction/HBaseAsyncPostCommitter.java | 67 -
.../com/yahoo/omid/transaction/HBaseCellId.java | 80 --
.../HBaseOmidClientConfiguration.java | 170 ---
.../transaction/HBaseSyncPostCommitter.java | 125 --
.../omid/transaction/HBaseTransaction.java | 79 --
.../transaction/HBaseTransactionClient.java | 24 -
.../transaction/HBaseTransactionManager.java | 292 ----
.../java/com/yahoo/omid/transaction/TTable.java | 770 -----------
.../transaction/TTableCellGetterAdapter.java | 38 -
.../apache/omid/transaction/ColumnWrapper.java | 64 +
.../transaction/HBaseAsyncPostCommitter.java | 67 +
.../apache/omid/transaction/HBaseCellId.java | 80 ++
.../HBaseOmidClientConfiguration.java | 170 +++
.../transaction/HBaseSyncPostCommitter.java | 125 ++
.../omid/transaction/HBaseTransaction.java | 79 ++
.../transaction/HBaseTransactionClient.java | 24 +
.../transaction/HBaseTransactionManager.java | 292 ++++
.../org/apache/omid/transaction/TTable.java | 770 +++++++++++
.../transaction/TTableCellGetterAdapter.java | 38 +
.../default-hbase-omid-client-config.yml | 4 +-
.../yahoo/omid/transaction/OmidTestBase.java | 257 ----
.../TestAsynchronousPostCommitter.java | 323 -----
.../yahoo/omid/transaction/TestAutoFlush.java | 62 -
.../omid/transaction/TestBasicTransaction.java | 440 ------
.../yahoo/omid/transaction/TestCellUtils.java | 230 ----
.../omid/transaction/TestColumnIterator.java | 96 --
.../yahoo/omid/transaction/TestDeletion.java | 299 ----
.../TestEndToEndScenariosWithHA.java | 412 ------
.../com/yahoo/omid/transaction/TestFilters.java | 191 ---
.../TestHALeaseManagementModule.java | 75 -
.../TestHBaseOmidClientConfiguration.java | 43 -
.../transaction/TestHBaseTransactionClient.java | 465 -------
.../TestHBaseTransactionManager.java | 102 --
.../yahoo/omid/transaction/TestMultiplePut.java | 134 --
.../yahoo/omid/transaction/TestReadPath.java | 105 --
.../yahoo/omid/transaction/TestShadowCells.java | 508 -------
.../transaction/TestSingleColumnFamily.java | 108 --
.../yahoo/omid/transaction/TestTSOModule.java | 98 --
.../omid/transaction/TestTTableBehaviour.java | 155 ---
.../transaction/TestTransactionCleanup.java | 119 --
.../transaction/TestTransactionConflict.java | 308 -----
.../omid/transaction/TestTxMgrFailover.java | 353 -----
.../yahoo/omid/transaction/TestUpdateScan.java | 218 ---
.../apache/omid/transaction/OmidTestBase.java | 257 ++++
.../TestAsynchronousPostCommitter.java | 323 +++++
.../apache/omid/transaction/TestAutoFlush.java | 62 +
.../omid/transaction/TestBasicTransaction.java | 440 ++++++
.../apache/omid/transaction/TestCellUtils.java | 230 ++++
.../omid/transaction/TestColumnIterator.java | 96 ++
.../apache/omid/transaction/TestDeletion.java | 299 ++++
.../TestEndToEndScenariosWithHA.java | 412 ++++++
.../apache/omid/transaction/TestFilters.java | 191 +++
.../TestHALeaseManagementModule.java | 75 +
.../TestHBaseOmidClientConfiguration.java | 43 +
.../transaction/TestHBaseTransactionClient.java | 465 +++++++
.../TestHBaseTransactionManager.java | 102 ++
.../omid/transaction/TestMultiplePut.java | 134 ++
.../apache/omid/transaction/TestReadPath.java | 105 ++
.../omid/transaction/TestShadowCells.java | 508 +++++++
.../transaction/TestSingleColumnFamily.java | 108 ++
.../apache/omid/transaction/TestTSOModule.java | 98 ++
.../omid/transaction/TestTTableBehaviour.java | 155 +++
.../transaction/TestTransactionCleanup.java | 119 ++
.../transaction/TestTransactionConflict.java | 308 +++++
.../omid/transaction/TestTxMgrFailover.java | 353 +++++
.../apache/omid/transaction/TestUpdateScan.java | 218 +++
.../src/test/resources/log4j.properties | 13 +-
.../resources/test-hbase-omid-client-config.yml | 18 +-
hbase-commit-table/pom.xml | 12 +-
.../DefaultHBaseCommitTableStorageModule.java | 89 --
.../committable/hbase/HBaseCommitTable.java | 422 ------
.../hbase/HBaseCommitTableConfig.java | 82 --
.../hbase/HBaseCommitTableStorageModule.java | 37 -
.../DefaultHBaseCommitTableStorageModule.java | 89 ++
.../committable/hbase/HBaseCommitTable.java | 422 ++++++
.../hbase/HBaseCommitTableConfig.java | 82 ++
.../hbase/HBaseCommitTableStorageModule.java | 37 +
.../committable/hbase/TestHBaseCommitTable.java | 298 ----
.../committable/hbase/TestHBaseCommitTable.java | 298 ++++
hbase-common/pom.xml | 6 +-
.../java/com/yahoo/omid/HBaseConfigModule.java | 83 --
.../omid/committable/hbase/KeyGenerator.java | 29 -
.../hbase/KeyGeneratorImplementations.java | 160 ---
.../omid/committable/hbase/RegionSplitter.java | 259 ----
.../com/yahoo/omid/tools/hbase/HBaseLogin.java | 38 -
.../omid/tools/hbase/SecureHBaseConfig.java | 55 -
.../com/yahoo/omid/transaction/CellInfo.java | 59 -
.../com/yahoo/omid/transaction/CellUtils.java | 406 ------
.../java/org/apache/omid/HBaseConfigModule.java | 83 ++
.../omid/committable/hbase/KeyGenerator.java | 29 +
.../hbase/KeyGeneratorImplementations.java | 160 +++
.../omid/committable/hbase/RegionSplitter.java | 258 ++++
.../org/apache/omid/tools/hbase/HBaseLogin.java | 38 +
.../omid/tools/hbase/SecureHBaseConfig.java | 55 +
.../org/apache/omid/transaction/CellInfo.java | 59 +
.../org/apache/omid/transaction/CellUtils.java | 406 ++++++
.../hbase/TestHBaseCommitTableKey.java | 53 -
.../hbase/TestHBaseCommitTableKey.java | 53 +
hbase-coprocessor/pom.xml | 16 +-
.../yahoo/omid/transaction/CompactorUtil.java | 107 --
.../transaction/HRegionCellGetterAdapter.java | 39 -
.../yahoo/omid/transaction/OmidCompactor.java | 140 --
.../hbase/regionserver/CompactorScanner.java | 14 +-
.../apache/omid/transaction/CompactorUtil.java | 107 ++
.../transaction/HRegionCellGetterAdapter.java | 39 +
.../apache/omid/transaction/OmidCompactor.java | 140 ++
.../TSOForHBaseCompactorTestModule.java | 120 --
.../yahoo/omid/transaction/TestCompaction.java | 1299 ------------------
.../omid/transaction/TestCompactorScanner.java | 106 --
.../TSOForHBaseCompactorTestModule.java | 120 ++
.../apache/omid/transaction/TestCompaction.java | 1299 ++++++++++++++++++
.../omid/transaction/TestCompactorScanner.java | 106 ++
hbase-coprocessor/src/test/resources/log4j.xml | 4 +-
hbase-shims/hbase-0/pom.xml | 2 +-
.../main/java/com/yahoo/omid/HBaseShims.java | 59 -
.../main/java/org/apache/omid/HBaseShims.java | 59 +
hbase-shims/hbase-1/pom.xml | 2 +-
.../main/java/com/yahoo/omid/HBaseShims.java | 58 -
.../main/java/org/apache/omid/HBaseShims.java | 58 +
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 10 +-
.../omid/tools/hbase/OmidTableManager.java | 203 ---
.../omid/tools/hbase/OmidTableManager.java | 203 +++
.../omid/tools/hbase/TestOmidTableManager.java | 119 --
.../omid/tools/hbase/TestOmidTableManager.java | 119 ++
metrics/pom.xml | 2 +-
.../omid/metrics/AbstractMetricsConfig.java | 43 -
.../java/com/yahoo/omid/metrics/Counter.java | 46 -
.../main/java/com/yahoo/omid/metrics/Gauge.java | 28 -
.../java/com/yahoo/omid/metrics/Histogram.java | 36 -
.../main/java/com/yahoo/omid/metrics/Meter.java | 34 -
.../java/com/yahoo/omid/metrics/Metric.java | 25 -
.../com/yahoo/omid/metrics/MetricsProvider.java | 41 -
.../com/yahoo/omid/metrics/MetricsRegistry.java | 61 -
.../yahoo/omid/metrics/MetricsRegistryMap.java | 105 --
.../com/yahoo/omid/metrics/MetricsUtils.java | 47 -
.../yahoo/omid/metrics/NullMetricsProvider.java | 135 --
.../main/java/com/yahoo/omid/metrics/Timer.java | 28 -
.../omid/metrics/AbstractMetricsConfig.java | 43 +
.../java/org/apache/omid/metrics/Counter.java | 46 +
.../java/org/apache/omid/metrics/Gauge.java | 28 +
.../java/org/apache/omid/metrics/Histogram.java | 36 +
.../java/org/apache/omid/metrics/Meter.java | 34 +
.../java/org/apache/omid/metrics/Metric.java | 25 +
.../apache/omid/metrics/MetricsProvider.java | 41 +
.../apache/omid/metrics/MetricsRegistry.java | 61 +
.../apache/omid/metrics/MetricsRegistryMap.java | 105 ++
.../org/apache/omid/metrics/MetricsUtils.java | 47 +
.../omid/metrics/NullMetricsProvider.java | 135 ++
.../java/org/apache/omid/metrics/Timer.java | 28 +
pom.xml | 25 +-
statemachine/pom.xml | 2 +-
.../com/yahoo/statemachine/StateMachine.java | 256 ----
.../statemachine/StateMachineLogParser.java | 163 ---
.../org/apache/statemachine/StateMachine.java | 256 ++++
.../statemachine/StateMachineLogParser.java | 163 +++
.../yahoo/statemachine/TestStateMachine.java | 100 --
.../apache/statemachine/TestStateMachine.java | 100 ++
timestamp-storage/pom.xml | 10 +-
.../DefaultHBaseTimestampStorageModule.java | 81 --
.../DefaultZKTimestampStorageModule.java | 57 -
.../storage/HBaseTimestampStorage.java | 89 --
.../storage/HBaseTimestampStorageConfig.java | 60 -
.../storage/HBaseTimestampStorageModule.java | 36 -
.../timestamp/storage/TimestampStorage.java | 28 -
.../yahoo/omid/timestamp/storage/ZKModule.java | 49 -
.../timestamp/storage/ZKTimestampPaths.java | 28 -
.../timestamp/storage/ZKTimestampStorage.java | 90 --
.../storage/ZKTimestampStorageModule.java | 37 -
.../DefaultHBaseTimestampStorageModule.java | 81 ++
.../DefaultZKTimestampStorageModule.java | 57 +
.../storage/HBaseTimestampStorage.java | 89 ++
.../storage/HBaseTimestampStorageConfig.java | 60 +
.../storage/HBaseTimestampStorageModule.java | 36 +
.../timestamp/storage/TimestampStorage.java | 28 +
.../apache/omid/timestamp/storage/ZKModule.java | 49 +
.../timestamp/storage/ZKTimestampPaths.java | 28 +
.../timestamp/storage/ZKTimestampStorage.java | 90 ++
.../storage/ZKTimestampStorageModule.java | 37 +
.../storage/TestHBaseTimestampStorage.java | 136 --
.../storage/TestZKTimestampStorage.java | 241 ----
.../storage/TestHBaseTimestampStorage.java | 136 ++
.../storage/TestZKTimestampStorage.java | 241 ++++
.../src/test/resources/log4j.properties | 15 +-
transaction-client/pom.xml | 12 +-
.../omid/transaction/AbstractTransaction.java | 217 ---
.../transaction/AbstractTransactionManager.java | 502 -------
.../transaction/CommitTimestampLocator.java | 38 -
.../omid/transaction/PostCommitActions.java | 41 -
.../omid/transaction/RollbackException.java | 35 -
.../com/yahoo/omid/transaction/Transaction.java | 76 -
.../omid/transaction/TransactionException.java | 31 -
.../omid/transaction/TransactionManager.java | 66 -
.../TransactionManagerException.java | 32 -
.../yahoo/omid/tso/client/AbortException.java | 27 -
.../java/com/yahoo/omid/tso/client/CellId.java | 24 -
.../yahoo/omid/tso/client/ClosingException.java | 28 -
.../omid/tso/client/ConnectionException.java | 30 -
.../omid/tso/client/ForwardingTSOFuture.java | 64 -
.../tso/client/HandshakeFailedException.java | 28 -
.../yahoo/omid/tso/client/MockTSOClient.java | 102 --
.../yahoo/omid/tso/client/NewTSOException.java | 27 -
.../tso/client/OmidClientConfiguration.java | 177 ---
.../tso/client/ServiceUnavailableException.java | 32 -
.../com/yahoo/omid/tso/client/TSOClient.java | 933 -------------
.../com/yahoo/omid/tso/client/TSOFuture.java | 25 -
.../com/yahoo/omid/tso/client/TSOProtocol.java | 56 -
.../yahoo/omid/tso/util/DummyCellIdImpl.java | 35 -
.../omid/transaction/AbstractTransaction.java | 217 +++
.../transaction/AbstractTransactionManager.java | 502 +++++++
.../transaction/CommitTimestampLocator.java | 38 +
.../omid/transaction/PostCommitActions.java | 41 +
.../omid/transaction/RollbackException.java | 35 +
.../apache/omid/transaction/Transaction.java | 76 +
.../omid/transaction/TransactionException.java | 31 +
.../omid/transaction/TransactionManager.java | 66 +
.../TransactionManagerException.java | 32 +
.../apache/omid/tso/client/AbortException.java | 27 +
.../java/org/apache/omid/tso/client/CellId.java | 24 +
.../omid/tso/client/ClosingException.java | 28 +
.../omid/tso/client/ConnectionException.java | 30 +
.../omid/tso/client/ForwardingTSOFuture.java | 64 +
.../tso/client/HandshakeFailedException.java | 28 +
.../apache/omid/tso/client/MockTSOClient.java | 102 ++
.../apache/omid/tso/client/NewTSOException.java | 27 +
.../tso/client/OmidClientConfiguration.java | 177 +++
.../tso/client/ServiceUnavailableException.java | 32 +
.../org/apache/omid/tso/client/TSOClient.java | 933 +++++++++++++
.../org/apache/omid/tso/client/TSOFuture.java | 25 +
.../org/apache/omid/tso/client/TSOProtocol.java | 56 +
.../apache/omid/tso/util/DummyCellIdImpl.java | 35 +
.../src/main/resources/omid-client-config.yml | 6 +-
.../omid/tso/client/TestMockTSOClient.java | 72 -
.../tso/client/TestOmidClientConfiguration.java | 32 -
.../omid/tso/client/TestMockTSOClient.java | 72 +
.../tso/client/TestOmidClientConfiguration.java | 32 +
tso-server/bin/omid.sh | 6 +-
tso-server/maven/assembly/bin.xml | 6 +-
tso-server/pom.xml | 20 +-
.../src/main/java/com/yahoo/omid/tso/Cache.java | 32 -
.../com/yahoo/omid/tso/CacheEvaluation.java | 105 --
.../java/com/yahoo/omid/tso/CommitHashMap.java | 81 --
.../com/yahoo/omid/tso/DisruptorModule.java | 36 -
.../yahoo/omid/tso/FatalExceptionHandler.java | 51 -
.../yahoo/omid/tso/HALeaseManagementModule.java | 81 --
.../tso/InMemoryCommitTableStorageModule.java | 35 -
.../tso/InMemoryTimestampStorageModule.java | 35 -
.../com/yahoo/omid/tso/LeaseManagement.java | 52 -
.../java/com/yahoo/omid/tso/LeaseManager.java | 310 -----
.../main/java/com/yahoo/omid/tso/LongCache.java | 85 --
.../java/com/yahoo/omid/tso/MockPanicker.java | 30 -
.../com/yahoo/omid/tso/MonitoringContext.java | 77 --
.../yahoo/omid/tso/NetworkInterfaceUtils.java | 126 --
.../main/java/com/yahoo/omid/tso/Panicker.java | 22 -
.../yahoo/omid/tso/PausableTimestampOracle.java | 70 -
.../yahoo/omid/tso/PersistenceProcessor.java | 30 -
.../omid/tso/PersistenceProcessorImpl.java | 431 ------
.../java/com/yahoo/omid/tso/ReplyProcessor.java | 43 -
.../com/yahoo/omid/tso/ReplyProcessorImpl.java | 236 ----
.../com/yahoo/omid/tso/RequestProcessor.java | 31 -
.../yahoo/omid/tso/RequestProcessorImpl.java | 309 -----
.../java/com/yahoo/omid/tso/RetryProcessor.java | 24 -
.../com/yahoo/omid/tso/RetryProcessorImpl.java | 185 ---
.../com/yahoo/omid/tso/SystemExitPanicker.java | 32 -
.../com/yahoo/omid/tso/TSOChannelHandler.java | 285 ----
.../main/java/com/yahoo/omid/tso/TSOModule.java | 67 -
.../main/java/com/yahoo/omid/tso/TSOServer.java | 131 --
.../com/yahoo/omid/tso/TSOServerConfig.java | 176 ---
.../com/yahoo/omid/tso/TSOStateManager.java | 95 --
.../com/yahoo/omid/tso/TSOStateManagerImpl.java | 78 --
.../com/yahoo/omid/tso/TimestampOracle.java | 34 -
.../com/yahoo/omid/tso/TimestampOracleImpl.java | 169 ---
.../com/yahoo/omid/tso/TsoServerDaemon.java | 60 -
.../omid/tso/VoidLeaseManagementModule.java | 92 --
.../com/yahoo/omid/tso/VoidLeaseManager.java | 53 -
.../main/java/org/apache/omid/tso/Cache.java | 32 +
.../org/apache/omid/tso/CacheEvaluation.java | 105 ++
.../java/org/apache/omid/tso/CommitHashMap.java | 81 ++
.../org/apache/omid/tso/DisruptorModule.java | 36 +
.../apache/omid/tso/FatalExceptionHandler.java | 51 +
.../omid/tso/HALeaseManagementModule.java | 81 ++
.../tso/InMemoryCommitTableStorageModule.java | 35 +
.../tso/InMemoryTimestampStorageModule.java | 35 +
.../org/apache/omid/tso/LeaseManagement.java | 52 +
.../java/org/apache/omid/tso/LeaseManager.java | 310 +++++
.../java/org/apache/omid/tso/LongCache.java | 85 ++
.../java/org/apache/omid/tso/MockPanicker.java | 30 +
.../org/apache/omid/tso/MonitoringContext.java | 77 ++
.../apache/omid/tso/NetworkInterfaceUtils.java | 126 ++
.../main/java/org/apache/omid/tso/Panicker.java | 22 +
.../omid/tso/PausableTimestampOracle.java | 70 +
.../apache/omid/tso/PersistenceProcessor.java | 30 +
.../omid/tso/PersistenceProcessorImpl.java | 431 ++++++
.../org/apache/omid/tso/ReplyProcessor.java | 43 +
.../org/apache/omid/tso/ReplyProcessorImpl.java | 236 ++++
.../org/apache/omid/tso/RequestProcessor.java | 31 +
.../apache/omid/tso/RequestProcessorImpl.java | 309 +++++
.../org/apache/omid/tso/RetryProcessor.java | 24 +
.../org/apache/omid/tso/RetryProcessorImpl.java | 185 +++
.../org/apache/omid/tso/SystemExitPanicker.java | 32 +
.../org/apache/omid/tso/TSOChannelHandler.java | 285 ++++
.../java/org/apache/omid/tso/TSOModule.java | 67 +
.../java/org/apache/omid/tso/TSOServer.java | 131 ++
.../org/apache/omid/tso/TSOServerConfig.java | 176 +++
.../org/apache/omid/tso/TSOStateManager.java | 95 ++
.../apache/omid/tso/TSOStateManagerImpl.java | 78 ++
.../org/apache/omid/tso/TimestampOracle.java | 34 +
.../apache/omid/tso/TimestampOracleImpl.java | 169 +++
.../org/apache/omid/tso/TsoServerDaemon.java | 60 +
.../omid/tso/VoidLeaseManagementModule.java | 92 ++
.../org/apache/omid/tso/VoidLeaseManager.java | 53 +
tso-server/src/main/resources/default-omid.yml | 74 +-
tso-server/src/main/resources/log4j.properties | 13 +-
.../src/test/java/com/yahoo/omid/TestUtils.java | 114 --
.../yahoo/omid/tso/PausableLeaseManager.java | 117 --
.../yahoo/omid/tso/ProgrammableTSOServer.java | 287 ----
.../java/com/yahoo/omid/tso/TSOMockModule.java | 76 -
.../com/yahoo/omid/tso/TSOServerConfigTest.java | 29 -
.../test/java/com/yahoo/omid/tso/TestBatch.java | 158 ---
.../com/yahoo/omid/tso/TestLeaseManager.java | 418 ------
.../java/com/yahoo/omid/tso/TestLongCache.java | 122 --
.../java/com/yahoo/omid/tso/TestPanicker.java | 163 ---
.../omid/tso/TestPersistenceProcessor.java | 221 ---
.../yahoo/omid/tso/TestRequestProcessor.java | 186 ---
.../com/yahoo/omid/tso/TestRetryProcessor.java | 126 --
.../omid/tso/TestTSOChannelHandlerNetty.java | 321 -----
.../com/yahoo/omid/tso/TestTSOStateManager.java | 121 --
.../com/yahoo/omid/tso/TestTimestampOracle.java | 117 --
.../omid/tso/client/TSOClientAccessor.java | 29 -
.../yahoo/omid/tso/client/TSOClientOneShot.java | 63 -
.../com/yahoo/omid/tso/client/TSOClientRaw.java | 150 --
...tionOfTSOClientServerBasicFunctionality.java | 233 ----
.../client/TestTSOClientConnectionToTSO.java | 284 ----
...stTSOClientRequestAndResponseBehaviours.java | 423 ------
.../client/TestTSOClientResponseHandling.java | 111 --
.../tso/client/TestUnconnectedTSOClient.java | 83 --
.../test/java/org/apache/omid/TestUtils.java | 114 ++
.../apache/omid/tso/PausableLeaseManager.java | 117 ++
.../apache/omid/tso/ProgrammableTSOServer.java | 287 ++++
.../java/org/apache/omid/tso/TSOMockModule.java | 76 +
.../apache/omid/tso/TSOServerConfigTest.java | 29 +
.../java/org/apache/omid/tso/TestBatch.java | 158 +++
.../org/apache/omid/tso/TestLeaseManager.java | 418 ++++++
.../java/org/apache/omid/tso/TestLongCache.java | 122 ++
.../java/org/apache/omid/tso/TestPanicker.java | 163 +++
.../omid/tso/TestPersistenceProcessor.java | 221 +++
.../apache/omid/tso/TestRequestProcessor.java | 186 +++
.../org/apache/omid/tso/TestRetryProcessor.java | 126 ++
.../omid/tso/TestTSOChannelHandlerNetty.java | 321 +++++
.../apache/omid/tso/TestTSOStateManager.java | 121 ++
.../apache/omid/tso/TestTimestampOracle.java | 117 ++
.../omid/tso/client/TSOClientAccessor.java | 29 +
.../omid/tso/client/TSOClientOneShot.java | 63 +
.../apache/omid/tso/client/TSOClientRaw.java | 150 ++
...tionOfTSOClientServerBasicFunctionality.java | 233 ++++
.../client/TestTSOClientConnectionToTSO.java | 284 ++++
...stTSOClientRequestAndResponseBehaviours.java | 423 ++++++
.../client/TestTSOClientResponseHandling.java | 111 ++
.../tso/client/TestUnconnectedTSOClient.java | 83 ++
tso-server/src/test/resources/log4j.properties | 15 +-
tso-server/src/test/resources/test-omid.yml | 8 +-
417 files changed, 26656 insertions(+), 26685 deletions(-)
----------------------------------------------------------------------
[52/52] [abbrv] incubator-omid git commit: Merge branch 'gh-master'
into apache-master
Posted by ik...@apache.org.
Merge branch 'gh-master' into apache-master
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/e5b4ca31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/e5b4ca31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/e5b4ca31
Branch: refs/heads/master
Commit: e5b4ca31597e1b19817b66c9752e8e76fc918ea0
Parents: 27964b2 0c0bcf5
Author: Igor Katkov <ka...@yahoo-inc.com>
Authored: Wed Apr 20 17:01:41 2016 -0700
Committer: Igor Katkov <ka...@yahoo-inc.com>
Committed: Wed Apr 20 17:01:41 2016 -0700
----------------------------------------------------------------------
.gitignore | 26 +
.travis.yml | 75 +
LICENSE.txt | 202 +++
README.md | 20 +
benchmarks/bin/benchmarks.sh | 47 +
benchmarks/bin/omid-env.sh | 2 +
benchmarks/conf/hbase-site.xml | 8 +
benchmarks/conf/log4j.xml | 15 +
benchmarks/maven/assembly/bin.xml | 66 +
benchmarks/pom.xml | 118 ++
.../hbase/HBaseCommitTableTester.java | 137 ++
.../apache/omid/benchmarks/tso/RawTxRunner.java | 301 ++++
.../omid/benchmarks/tso/TSOServerBenchmark.java | 138 ++
.../tso/TSOServerBenchmarkConfig.java | 160 ++
.../apache/omid/benchmarks/utils/Generator.java | 30 +
.../omid/benchmarks/utils/IntegerGenerator.java | 65 +
.../utils/ScrambledZipfianGenerator.java | 147 ++
.../omid/benchmarks/utils/UniformGenerator.java | 36 +
.../omid/benchmarks/utils/ZipfianGenerator.java | 292 ++++
.../default-tso-server-benchmark-config.yml | 68 +
bintray-settings.xml | 9 +
codahale-metrics/pom.xml | 65 +
.../omid/metrics/CodahaleMetricsConfig.java | 100 ++
.../omid/metrics/CodahaleMetricsProvider.java | 346 ++++
commit-table/pom.xml | 70 +
.../apache/omid/committable/CommitTable.java | 113 ++
.../omid/committable/InMemoryCommitTable.java | 143 ++
.../omid/committable/NullCommitTable.java | 91 +
.../omid/committable/NullCommitTableTest.java | 70 +
common/pom.xml | 134 ++
.../main/java/org/apache/omid/YAMLUtils.java | 81 +
.../main/java/org/apache/omid/zk/ZKUtils.java | 57 +
common/src/main/proto/TSOProto.proto | 67 +
.../java/org/apache/omid/YAMLUtilsTest.java | 50 +
common/src/test/resources/default-test.yml | 3 +
common/src/test/resources/test.yml | 2 +
doc/images/ModuleDependencies.graffle | 1606 ++++++++++++++++++
doc/images/ModuleDependencies.png | Bin 0 -> 66847 bytes
doc/images/omid-logo.png | Bin 0 -> 23473 bytes
doc/site/markdown/basic-algorithm.md | 29 +
doc/site/markdown/basic-concepts.md | 79 +
doc/site/markdown/basic-examples.md | 264 +++
doc/site/markdown/client-failure-management.md | 30 +
doc/site/markdown/coding-guide-and-style.md | 60 +
doc/site/markdown/index.md | 94 +
doc/site/markdown/mailing-lists.md | 13 +
doc/site/markdown/omid-components.md | 96 ++
doc/site/markdown/quickstart.md | 203 +++
doc/site/resources/css/site.css | 103 ++
doc/site/resources/images/about-omid.png | Bin 0 -> 3402 bytes
doc/site/resources/images/architecture.png | Bin 0 -> 77416 bytes
doc/site/resources/images/basic-alg.png | Bin 0 -> 30454 bytes
doc/site/resources/images/contact.png | Bin 0 -> 2899 bytes
doc/site/resources/images/getting-started.png | Bin 0 -> 4328 bytes
.../resources/images/omid-logo-transparent.png | Bin 0 -> 13867 bytes
doc/site/resources/images/omid-logo.png | Bin 0 -> 23473 bytes
.../resources/images/snapshot-isolation.png | Bin 0 -> 22535 bytes
.../images/technical-documentation.png | Bin 0 -> 2857 bytes
doc/site/site.xml | 85 +
examples/maven/assembly/assembly.xml | 61 +
examples/pom.xml | 143 ++
examples/run.sh | 98 ++
.../org/apache/omid/examples/BasicExample.java | 112 ++
.../omid/examples/ConfigurationExample.java | 131 ++
.../omid/examples/SnapshotIsolationExample.java | 174 ++
.../main/resources/hbase-omid-client-config.yml | 20 +
examples/src/main/resources/log4j.properties | 10 +
hbase-client/pom.xml | 189 +++
.../apache/omid/transaction/ColumnWrapper.java | 64 +
.../transaction/HBaseAsyncPostCommitter.java | 67 +
.../apache/omid/transaction/HBaseCellId.java | 80 +
.../HBaseOmidClientConfiguration.java | 170 ++
.../transaction/HBaseSyncPostCommitter.java | 125 ++
.../omid/transaction/HBaseTransaction.java | 79 +
.../transaction/HBaseTransactionClient.java | 24 +
.../transaction/HBaseTransactionManager.java | 292 ++++
.../org/apache/omid/transaction/TTable.java | 770 +++++++++
.../transaction/TTableCellGetterAdapter.java | 38 +
.../default-hbase-omid-client-config.yml | 8 +
.../apache/omid/transaction/OmidTestBase.java | 257 +++
.../TestAsynchronousPostCommitter.java | 323 ++++
.../apache/omid/transaction/TestAutoFlush.java | 62 +
.../omid/transaction/TestBasicTransaction.java | 440 +++++
.../apache/omid/transaction/TestCellUtils.java | 230 +++
.../omid/transaction/TestColumnIterator.java | 96 ++
.../apache/omid/transaction/TestDeletion.java | 299 ++++
.../TestEndToEndScenariosWithHA.java | 412 +++++
.../apache/omid/transaction/TestFilters.java | 191 +++
.../TestHALeaseManagementModule.java | 75 +
.../TestHBaseOmidClientConfiguration.java | 43 +
.../transaction/TestHBaseTransactionClient.java | 465 +++++
.../TestHBaseTransactionManager.java | 102 ++
.../omid/transaction/TestMultiplePut.java | 134 ++
.../apache/omid/transaction/TestReadPath.java | 105 ++
.../omid/transaction/TestShadowCells.java | 508 ++++++
.../transaction/TestSingleColumnFamily.java | 108 ++
.../apache/omid/transaction/TestTSOModule.java | 98 ++
.../omid/transaction/TestTTableBehaviour.java | 155 ++
.../transaction/TestTransactionCleanup.java | 119 ++
.../transaction/TestTransactionConflict.java | 308 ++++
.../omid/transaction/TestTxMgrFailover.java | 353 ++++
.../apache/omid/transaction/TestUpdateScan.java | 218 +++
.../src/test/resources/log4j.properties | 54 +
.../resources/test-hbase-omid-client-config.yml | 34 +
hbase-commit-table/pom.xml | 168 ++
.../DefaultHBaseCommitTableStorageModule.java | 89 +
.../committable/hbase/HBaseCommitTable.java | 422 +++++
.../hbase/HBaseCommitTableConfig.java | 82 +
.../hbase/HBaseCommitTableStorageModule.java | 37 +
.../committable/hbase/TestHBaseCommitTable.java | 298 ++++
hbase-common/pom.xml | 109 ++
.../java/org/apache/omid/HBaseConfigModule.java | 83 +
.../omid/committable/hbase/KeyGenerator.java | 29 +
.../hbase/KeyGeneratorImplementations.java | 160 ++
.../omid/committable/hbase/RegionSplitter.java | 258 +++
.../org/apache/omid/tools/hbase/HBaseLogin.java | 38 +
.../omid/tools/hbase/SecureHBaseConfig.java | 55 +
.../org/apache/omid/transaction/CellInfo.java | 59 +
.../org/apache/omid/transaction/CellUtils.java | 406 +++++
.../hbase/TestHBaseCommitTableKey.java | 53 +
hbase-coprocessor/.gitignore | 1 +
hbase-coprocessor/pom.xml | 174 ++
.../hbase/regionserver/CompactorScanner.java | 286 ++++
.../apache/omid/transaction/CompactorUtil.java | 107 ++
.../transaction/HRegionCellGetterAdapter.java | 39 +
.../apache/omid/transaction/OmidCompactor.java | 140 ++
.../TSOForHBaseCompactorTestModule.java | 120 ++
.../apache/omid/transaction/TestCompaction.java | 1299 ++++++++++++++
.../omid/transaction/TestCompactorScanner.java | 106 ++
hbase-coprocessor/src/test/resources/log4j.xml | 68 +
hbase-shims/hbase-0/pom.xml | 19 +
.../hadoop/hbase/regionserver/Region.java | 47 +
.../hbase/regionserver/ScannerContext.java | 28 +
.../main/java/org/apache/omid/HBaseShims.java | 59 +
hbase-shims/hbase-1/pom.xml | 19 +
.../main/java/org/apache/omid/HBaseShims.java | 58 +
hbase-shims/pom.xml | 83 +
hbase-tools/pom.xml | 125 ++
.../omid/tools/hbase/OmidTableManager.java | 203 +++
.../omid/tools/hbase/TestOmidTableManager.java | 119 ++
metrics/pom.xml | 70 +
.../omid/metrics/AbstractMetricsConfig.java | 43 +
.../java/org/apache/omid/metrics/Counter.java | 46 +
.../java/org/apache/omid/metrics/Gauge.java | 28 +
.../java/org/apache/omid/metrics/Histogram.java | 36 +
.../java/org/apache/omid/metrics/Meter.java | 34 +
.../java/org/apache/omid/metrics/Metric.java | 25 +
.../apache/omid/metrics/MetricsProvider.java | 41 +
.../apache/omid/metrics/MetricsRegistry.java | 61 +
.../apache/omid/metrics/MetricsRegistryMap.java | 105 ++
.../org/apache/omid/metrics/MetricsUtils.java | 47 +
.../omid/metrics/NullMetricsProvider.java | 135 ++
.../java/org/apache/omid/metrics/Timer.java | 28 +
misc/header.txt | 15 +
misc/omid_checks.xml | 211 +++
pom.xml | 607 +++++++
statemachine/pom.xml | 56 +
.../org/apache/statemachine/StateMachine.java | 256 +++
.../statemachine/StateMachineLogParser.java | 163 ++
.../apache/statemachine/TestStateMachine.java | 100 ++
timestamp-storage/pom.xml | 155 ++
.../DefaultHBaseTimestampStorageModule.java | 81 +
.../DefaultZKTimestampStorageModule.java | 57 +
.../storage/HBaseTimestampStorage.java | 89 +
.../storage/HBaseTimestampStorageConfig.java | 60 +
.../storage/HBaseTimestampStorageModule.java | 36 +
.../timestamp/storage/TimestampStorage.java | 28 +
.../apache/omid/timestamp/storage/ZKModule.java | 49 +
.../timestamp/storage/ZKTimestampPaths.java | 28 +
.../timestamp/storage/ZKTimestampStorage.java | 90 +
.../storage/ZKTimestampStorageModule.java | 37 +
.../storage/TestHBaseTimestampStorage.java | 136 ++
.../storage/TestZKTimestampStorage.java | 241 +++
.../src/test/resources/log4j.properties | 58 +
transaction-client/pom.xml | 117 ++
.../omid/transaction/AbstractTransaction.java | 217 +++
.../transaction/AbstractTransactionManager.java | 502 ++++++
.../transaction/CommitTimestampLocator.java | 38 +
.../omid/transaction/PostCommitActions.java | 41 +
.../omid/transaction/RollbackException.java | 35 +
.../apache/omid/transaction/Transaction.java | 76 +
.../omid/transaction/TransactionException.java | 31 +
.../omid/transaction/TransactionManager.java | 66 +
.../TransactionManagerException.java | 32 +
.../apache/omid/tso/client/AbortException.java | 27 +
.../java/org/apache/omid/tso/client/CellId.java | 24 +
.../omid/tso/client/ClosingException.java | 28 +
.../omid/tso/client/ConnectionException.java | 30 +
.../omid/tso/client/ForwardingTSOFuture.java | 64 +
.../tso/client/HandshakeFailedException.java | 28 +
.../apache/omid/tso/client/MockTSOClient.java | 102 ++
.../apache/omid/tso/client/NewTSOException.java | 27 +
.../tso/client/OmidClientConfiguration.java | 177 ++
.../tso/client/ServiceUnavailableException.java | 32 +
.../org/apache/omid/tso/client/TSOClient.java | 933 ++++++++++
.../org/apache/omid/tso/client/TSOFuture.java | 25 +
.../org/apache/omid/tso/client/TSOProtocol.java | 56 +
.../apache/omid/tso/util/DummyCellIdImpl.java | 35 +
.../src/main/resources/omid-client-config.yml | 39 +
.../omid/tso/client/TestMockTSOClient.java | 72 +
.../tso/client/TestOmidClientConfiguration.java | 32 +
tso-server/bin/omid.sh | 71 +
tso-server/conf/bench.log4j.xml | 15 +
tso-server/conf/hbase-site.xml | 6 +
tso-server/conf/log4j.xml | 34 +
tso-server/conf/omid-env.sh | 2 +
tso-server/conf/omid.yml | 1 +
tso-server/maven/assembly/bin.xml | 66 +
tso-server/pom.xml | 299 ++++
.../main/java/org/apache/omid/tso/Cache.java | 32 +
.../org/apache/omid/tso/CacheEvaluation.java | 105 ++
.../java/org/apache/omid/tso/CommitHashMap.java | 81 +
.../org/apache/omid/tso/DisruptorModule.java | 36 +
.../apache/omid/tso/FatalExceptionHandler.java | 51 +
.../omid/tso/HALeaseManagementModule.java | 81 +
.../tso/InMemoryCommitTableStorageModule.java | 35 +
.../tso/InMemoryTimestampStorageModule.java | 35 +
.../org/apache/omid/tso/LeaseManagement.java | 52 +
.../java/org/apache/omid/tso/LeaseManager.java | 310 ++++
.../java/org/apache/omid/tso/LongCache.java | 85 +
.../java/org/apache/omid/tso/MockPanicker.java | 30 +
.../org/apache/omid/tso/MonitoringContext.java | 77 +
.../apache/omid/tso/NetworkInterfaceUtils.java | 126 ++
.../main/java/org/apache/omid/tso/Panicker.java | 22 +
.../omid/tso/PausableTimestampOracle.java | 70 +
.../apache/omid/tso/PersistenceProcessor.java | 30 +
.../omid/tso/PersistenceProcessorImpl.java | 431 +++++
.../org/apache/omid/tso/ReplyProcessor.java | 43 +
.../org/apache/omid/tso/ReplyProcessorImpl.java | 236 +++
.../org/apache/omid/tso/RequestProcessor.java | 31 +
.../apache/omid/tso/RequestProcessorImpl.java | 309 ++++
.../org/apache/omid/tso/RetryProcessor.java | 24 +
.../org/apache/omid/tso/RetryProcessorImpl.java | 185 ++
.../org/apache/omid/tso/SystemExitPanicker.java | 32 +
.../org/apache/omid/tso/TSOChannelHandler.java | 285 ++++
.../java/org/apache/omid/tso/TSOModule.java | 67 +
.../java/org/apache/omid/tso/TSOServer.java | 131 ++
.../org/apache/omid/tso/TSOServerConfig.java | 176 ++
.../org/apache/omid/tso/TSOStateManager.java | 95 ++
.../apache/omid/tso/TSOStateManagerImpl.java | 78 +
.../org/apache/omid/tso/TimestampOracle.java | 34 +
.../apache/omid/tso/TimestampOracleImpl.java | 169 ++
.../org/apache/omid/tso/TsoServerDaemon.java | 60 +
.../omid/tso/VoidLeaseManagementModule.java | 92 +
.../org/apache/omid/tso/VoidLeaseManager.java | 53 +
tso-server/src/main/resources/default-omid.yml | 145 ++
tso-server/src/main/resources/log4j.properties | 55 +
.../test/java/org/apache/omid/TestUtils.java | 114 ++
.../apache/omid/tso/PausableLeaseManager.java | 117 ++
.../apache/omid/tso/ProgrammableTSOServer.java | 287 ++++
.../java/org/apache/omid/tso/TSOMockModule.java | 76 +
.../apache/omid/tso/TSOServerConfigTest.java | 29 +
.../java/org/apache/omid/tso/TestBatch.java | 158 ++
.../org/apache/omid/tso/TestLeaseManager.java | 418 +++++
.../java/org/apache/omid/tso/TestLongCache.java | 122 ++
.../java/org/apache/omid/tso/TestPanicker.java | 163 ++
.../omid/tso/TestPersistenceProcessor.java | 221 +++
.../apache/omid/tso/TestRequestProcessor.java | 186 ++
.../org/apache/omid/tso/TestRetryProcessor.java | 126 ++
.../omid/tso/TestTSOChannelHandlerNetty.java | 321 ++++
.../apache/omid/tso/TestTSOStateManager.java | 121 ++
.../apache/omid/tso/TestTimestampOracle.java | 117 ++
.../omid/tso/client/TSOClientAccessor.java | 29 +
.../omid/tso/client/TSOClientOneShot.java | 63 +
.../apache/omid/tso/client/TSOClientRaw.java | 150 ++
...tionOfTSOClientServerBasicFunctionality.java | 233 +++
.../client/TestTSOClientConnectionToTSO.java | 284 ++++
...stTSOClientRequestAndResponseBehaviours.java | 423 +++++
.../client/TestTSOClientResponseHandling.java | 111 ++
.../tso/client/TestUnconnectedTSOClient.java | 83 +
tso-server/src/test/resources/log4j.properties | 59 +
tso-server/src/test/resources/test-omid.yml | 21 +
272 files changed, 33959 insertions(+)
----------------------------------------------------------------------
[46/52] [abbrv] incubator-omid git commit: Add infrastructure to
create Omid website
Posted by ik...@apache.org.
Add infrastructure to create Omid website
The site is generated with the Apache site plugin and uses the
fluido skin, which is based on Twitter's bootstrap. The main pom file
contains its configuration and the site generation has been blocked
in the submodules, as they are not independent projects. All the site-related
configuration and documentation can be found in the "site" directory
and the generated static content to deploy that is generated with
mvn site:site and mvn site:staging is put to the "website" and "website-staging"
directories respectively.
When using mvn site:deploy, the website is directly pushed to the
Apache github repo by using the wagon-git extension (See wagon-git
extension in main pom.xml)
Change-Id: I55cb5bd9a0d1921433aec0ccdf2213f48221ebbe
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/05d69d9a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/05d69d9a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/05d69d9a
Branch: refs/heads/master
Commit: 05d69d9a0e6d0e3c2e5f0388d258889f330721b2
Parents: fd8ca18
Author: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Authored: Wed Apr 13 14:00:24 2016 -0700
Committer: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Committed: Tue Apr 19 13:51:57 2016 -0700
----------------------------------------------------------------------
.gitignore | 8 +-
benchmarks/pom.xml | 9 +
codahale-metrics/pom.xml | 16 ++
commit-table/pom.xml | 8 +
common/pom.xml | 8 +
doc/site/markdown/basic-algorithm.md | 29 ++
doc/site/markdown/basic-concepts.md | 79 ++++++
doc/site/markdown/basic-examples.md | 264 +++++++++++++++++++
doc/site/markdown/client-failure-management.md | 30 +++
doc/site/markdown/coding-guide-and-style.md | 56 ++++
doc/site/markdown/index.md | 94 +++++++
doc/site/markdown/mailing-lists.md | 13 +
doc/site/markdown/omid-components.md | 96 +++++++
doc/site/markdown/quickstart.md | 203 ++++++++++++++
doc/site/resources/css/site.css | 103 ++++++++
doc/site/resources/images/about-omid.png | Bin 0 -> 3402 bytes
doc/site/resources/images/architecture.png | Bin 0 -> 77416 bytes
doc/site/resources/images/basic-alg.png | Bin 0 -> 30454 bytes
doc/site/resources/images/contact.png | Bin 0 -> 2899 bytes
doc/site/resources/images/getting-started.png | Bin 0 -> 4328 bytes
.../resources/images/omid-logo-transparent.png | Bin 0 -> 13867 bytes
doc/site/resources/images/omid-logo.png | Bin 0 -> 23473 bytes
.../resources/images/snapshot-isolation.png | Bin 0 -> 22535 bytes
.../images/technical-documentation.png | Bin 0 -> 2857 bytes
doc/site/site.xml | 85 ++++++
examples/pom.xml | 8 +
hbase-client/pom.xml | 7 +
hbase-commit-table/pom.xml | 16 ++
hbase-common/pom.xml | 16 ++
hbase-coprocessor/pom.xml | 8 +
hbase-shims/pom.xml | 16 ++
hbase-tools/pom.xml | 16 ++
metrics/pom.xml | 16 ++
pom.xml | 111 +++++++-
statemachine/pom.xml | 16 ++
timestamp-storage/pom.xml | 16 ++
transaction-client/pom.xml | 16 ++
tso-server/pom.xml | 8 +
38 files changed, 1362 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 03517e6..c5d7a96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,12 @@ lib/
*~
*.swp
+# Generated website files
+generated-website/
+generated-website-staging/
+
# TestNG
*/test-output/
-common/src/main/java/org/apache/omid/proto/
+
+# Protobuf generated classes
+common/src/main/java/com/yahoo/omid/proto/
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index 8ec06db..1b8d49d 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -102,6 +102,15 @@
</configuration>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index a34204f..1fddbcd 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -46,4 +46,20 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index ad5c100..47afe60 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -55,6 +55,14 @@
</executions>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index d776e2b..3952b3d 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -120,6 +120,14 @@
</executions>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/basic-algorithm.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/basic-algorithm.md b/doc/site/markdown/basic-algorithm.md
new file mode 100644
index 0000000..4b0d4cd
--- /dev/null
+++ b/doc/site/markdown/basic-algorithm.md
@@ -0,0 +1,29 @@
+# Basic Transaction Algorithm
+
+A simplified version of the algorithm for performing transactional operations in a non-faulty scenario is depicted
+in the following figure:
+
+
+
+Initially, a user application (not shown in the figure for the sake of simplicity) starts a new transaction using
+the Transactional Client API, which in turn acquires a start timestamp from the TSO.
+
+Then, the user application logic, also through the Transactional Client API, performs a series of read and write
+operations in the snapshot provided by the transactional context recently acquired.
+
+In the case of reads, if the cells read have shadow cells and their value is in the snapshot for the transaction,
+their value is taken directly. If the commit timestamp is missed in the shadow cell, the Transactional Client will
+try to find it in the commit table. If the commit timestamp is found in there and the cell value is in the snapshot,
+the cell value is taken. However, if the commit timestamp is missed, the shadow cell is re-checked. If it still does
+not exist, the cell is ignored and another version that matches the snapshot will be retrieved from the datastore.
+
+When the application commits the transaction, the Transactional Client contacts the TSO in order to check the possible
+conflicts of the transaction. If the writeset does not conflict with other concurrent transactions, the transaction is
+committed, the TSO updates the commit table and replies back to the Transactional Client, which in turn, returns the
+control to the user application.
+
+Finally, the Transactional Client, on receiving the commit acknowledgement, updates the shadow cells with the required
+data. After this, it can also remove safely the entry added by the TSO in the commit table.
+
+In case the TSO detects conflicts, the transaction is aborted (not shown in the figure), the Transactional Client will
+clean up the data written to the datastore, and will inform the user application throwing a rollback exception.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/basic-concepts.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/basic-concepts.md b/doc/site/markdown/basic-concepts.md
new file mode 100644
index 0000000..2aaa35d
--- /dev/null
+++ b/doc/site/markdown/basic-concepts.md
@@ -0,0 +1,79 @@
+# Basic Concepts
+
+Omid started as a research project in 2011 at Yahoo Labs, with the aim of adding transactions to HBase.
+While Omid was designed with HBase in mind, we have been careful to not bind ourselves to a particular big
+datastore. As a consequence, Omid can be potentially adapted to be used with any other multiversioned key-value
+datastore with some additional extensions.
+
+In the next sections, we introduce two basic concepts that have driven Omid's design.
+
+## Locking vs Lock-Free
+
+When solving the **conflict detection** problem in transactional systems, there are two main approaches that
+transaction managers may implement: *locking* and *lock-free*.
+
+With a locking mechanism, the client managing the transaction attempts to acquire a lock on each cell in the writeset
+(the set of changes in data performed by the transaction) before committing. If it succeeds in acquiring the lock, and
+no other transaction has written to the cells since the start of the client's transaction, it can commit, as it knows
+that no other transaction can write to the cell after the lock has been acquired. Once the commit has completed, the
+client can release the locks on the cells. If the client fails to acquire the lock on any cell or some other transaction
+has written to it since the start of the transaction, then the transaction is aborted.
+
+For example, [Google Percolator](http://research.google.com/pubs/pub36726.html) uses a locking mechanism in its
+transactional system, and so do other subsequent academic and open source transaction managers for NoSQL datastores.
+
+Locking systems can be problematic in a distributed setting and in the presence of process failures. If a client fails
+while holding a lock another process must clean up the locks held by that client. However, it is impossible to detect
+reliably if a client has failed or is just being slow. Normally this is dealt with using timeouts, after which the
+client is invalidated and its locks released. However, the timeout must be high enough to ensure that valid clients
+are not invalidated.
+
+In lock-based systems, it is also possible for two transactions to repeatedly conflict with each other over multiple
+retries as there is no defined order in which locks must be acquired.
+
+In a lock-free implementation, this type of livelock is not possible. Transactions can always make progress. A lock-free
+implementation has a centralized conflict-detection mechanism. To commit a transaction, each transaction writeset is sent
+to this mechanism to check for conflicts with the writesets of other concurrent transactions. If none of those transaction
+has written to any of the cells of the writeset since the start of the transaction, the transaction can commit. Otherwise
+the transaction is aborted. Omid uses a lock-free mechanism for conflict detection.
+
+## Snapshot Isolation
+
+Isolation in ACID refers to the ability for multiple clients to act on a database without interfering with each other.
+There are various isolation levels, each making a different tradeoff between isolation and concurrent access to the database.
+Omid is only concerned with Snapshot Isolation.
+
+**[Snapshot Isolation (SI)](http://research.microsoft.com/pubs/69541/tr-95-51.pdf)** provides the users of a database with
+a consistent view of the database and the ability to detect write-write conficts between transactions. When a client starts a
+transaction, it is given a *"snapshot view"* of the database. A value read from a cell within this snapshot will always be
+the same, unless the transaction itself updates the cell.
+
+To successfully commit a transaction within snapshot isolation, the system must ensure that no other transaction has written to
+any of the transaction's written cells (that is, its writeset) since the start of the transaction.
+
+That is, transactions with SI guarantees should read from their own data snapshot, being a data snapshot:
+
+* Immutable
+* Identified by creation time
+* Containing values committed before creation time
+
+In Omid, transactions conflict iff:
+
+* Overlap in time, and...
+* ... write to the same cell of a particular row
+
+The following figure describes SI with an example:
+
+
+As depicted in the figure, transaction T2 overlaps in time with T1 and T3, but spatially:
+
+* T1 ∩ T2 = ∅
+* T2 ∩ T3 = { R4 }
+
+So, transactions T1 and T2, despite overlaping in time, they do not overlap in the elements they modify, so they do not conflict.
+However, transactions T2 and T3 have a conflict because they overlap both in space (both modify R4) and time. Finally, Transaction
+T4 does not has conflicts with other transactions.
+
+Compared to other isolation levels, SI offers good-enough consistency guarantees -although some anomalies may appear on data, e.g.
+write-skew- and performance, mainly due to the fact transactions are not aborted due to read-write conflicts. We took this into
+consideration and we decided to implement SI as the default isolation level for Omid.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/basic-examples.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/basic-examples.md b/doc/site/markdown/basic-examples.md
new file mode 100644
index 0000000..3a0b679
--- /dev/null
+++ b/doc/site/markdown/basic-examples.md
@@ -0,0 +1,264 @@
+# Code Examples
+
+This section guides the developer of Omid-based transactional applications over the different steps required to
+execute multi-row transactions on HBase and the different APIs involved in the process.
+
+## Transactional Contexts and Operations with Omid
+
+Applications requiring transactions will use the interfaces provided by the Omid _Transactional Client_. These interfaces
+are provided by the _TransactionManager_ and _TTable_ classes described below.
+
+When starting a transaction, the Omid _Transactional Client_ requests a _start timestamp_ from the _TSO_. This
+timestamp marks the beginning of a transactional context, that is, a transaction. From that point on, the client
+application can perform transactional read and write operations on the data source:
+
+* **Read Ops**. The client first checks if the cell has a _Shadow Cell (SC)_. If the cell has a _SC_ and the
+_commit timestamp_ is lower than the _start timestamp_ of the reading transaction, then the client can "see" the
+cell, that is, it's in its snapshot. If there is no _SC_, the _Commit Table (CT)_ is consulted to find the
+_commit timestamp_ of the cell. If the _commit timestamp_ does not exist in the _CT_, then the cell is assumed to
+below to an aborted transaction.
+* **Write Ops**. The client directly writes optimistically in the data source any data cells it wishes adding
+the _start timestamp_ as the version of each data cell. The metadata of each cell written is added also to the
+so-called _write-set_ of the transaction.
+
+To commit a transaction, the client sends the _write-set_ of the transaction to the _TSO_, which will check for
+conflicts in the transaction _write-set_ with other transactions executing concurrently:
+
+* **Conflicts**. The transaction is aborted and the _roll-back_ outcome is returned to the client.
+* **No conflicts**. The TSO assigns a _commit timestamp_ to the transaction, writes the mapping
+_start timestamp -> commit timestamp_ to the _CT_ and returns _commit_ as an outcome for the transaction to the client.
+
+On receiving the outcome for the transaction, the client:
+
+* **For roll-backs**, it clears the written cells.
+* **For commits**, it updates the _shadow cells_ for the cells in the _write-set_ and clears the entry for that
+transaction in the _CT_.
+
+The following sections show step-by-step the process for creating transactions in client applications with the Omid APIs.
+
+## Obtaining the Transaction Manager
+
+In order to use transactions, a client application needs to create an instance of the `TransactionManager` interface
+with is part of the Transactional Client described in the architecture. The current Omid version offers an
+implementation of a transaction manager for HBase. Please, make sure `core-site.xml` and `hbase-site.xml` HBase
+configuration files are present in the CLASSPATH of your client application.
+
+To create a transaction manager just add the following code to your application:
+
+```java
+
+ ...
+ TransactionManager tm = HBaseTransactionManager.newInstance();
+ ...
+
+```
+
+If nothing is specified, the HBase transaction manager instance is created with default configuration settings
+loaded from the `default-hbase-omid-client-config.yml` file included in the HBase Omid client jar. To change the
+client default settings, there are two possibilities:
+
+1) Put the specific configuration settings in a file named `hbase-omid-client-config.yml` and include it in the
+CLASSPATH. The file has the following format:
+
+```
+
+# HBase related
+commitTableName: MY_OWN_OMID_COMMIT_TABLE_NAME
+
+# TSO/ZK connection
+omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
+ connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType ZK
+ connectionString: "my_zk_cluster_conn_string"
+
+# Instrumentation
+metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
+
+```
+
+2) Create an instance of the `HBaseOmidClientConfiguration` class in the application code and pass it in the creation
+of the transaction manager instance:
+
+```java
+ ...
+ HBaseOmidClientConfiguration omidClientConfiguration = new HBaseOmidClientConfiguration();
+ omidClientConfiguration.setConnectionType(DIRECT);
+ omidClientConfiguration.setConnectionString("my_tso_server_host:54758");
+ omidClientConfiguration.setRetryDelayMs(3000);
+
+ TransactionManager tm = HBaseTransactionManager.newInstance(omidClientConfiguration);
+ ...
+```
+
+Please, refer to the [ConfigurationExample](https://github.com/yahoo/omid/blob/master/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java)
+in the source code to experiment with the configuration options.
+
+## Creating Transactions
+
+Once the `TransactionManager` is created, client applications can use its interface to demarcate transactional boundaries.
+
+In order to create a transaction the `TransactionManager.begin()` method is used:
+
+```java
+
+ ...
+ Transaction tx = tm.begin();
+ ...
+
+```
+
+The transaction manager will return an instance of the `Transaction` interface representing the recently created
+transaction. This instance is necessary to instruct the operations on the data source, in which transactional context
+they should operate (See next section).
+
+## Executing Transactional Operations
+
+In order to perform transaction operations on data, the client application requires to use a wrapper on the HBase's
+`HTableInterface` abstraction. The wrapper is called `TTable` and is also part of what is described as Transactional
+Client in the Omid architecture (See section [[About Omid]]). `TTable` basically offers the same interface as
+`HTableInterface` enhanced with a parameter representing the transactional context. As was previously described, a
+`Transaction` instance is obtained on return of `TransactionManager.begin()` method calls.
+
+```java
+
+ ...
+ TTable txTable = new TTable(conf, "EXAMPLE_TABLE");
+ ...
+
+```
+
+Once the access point to the data has been created, applications can use it to trigger transactional operations:
+
+```java
+
+ private final byte[] family = Bytes.toBytes("EXAMPLE_CF");
+ private final byte[] qualifier = Bytes.toBytes("foo");
+ ...
+
+ // Retrieve transactionally a specific cell
+ Get get = new Get(Bytes.toBytes("EXAMPLE_ROW");
+ get.add(family, qualifier);
+ Result txGetResult = txTable.get(tx, get);
+ ...
+
+ // Add a new cell value inside a transactional context
+ Put updatedRow = new Put(Bytes.toBytes("EXAMPLE_ROW");
+ updatedRow.add(family, qualifier, Bytes.toBytes("Another_value"));
+ txTable.put(tx, updatedRow);
+ ...
+
+```
+
+## Committing and Aborting Transactions
+
+Once the client application has finished reading/writting from/into the datasource, it must decide whether to make the
+changes visible or to discard them. In order to do this, it must instruct the Omid `TransactionManager` either to
+`commit()` or to `rollback()` the transactional context where the changes were produced. In case of commit, the TSO
+server will be notified to perform the SI validation phase. If the validation succeeds the changes will be visible to
+new transactions started from that point on. Otherwise, it will roll back the changes.
+
+In order to commit the data, the client application should do something like this:
+
+```java
+
+ ...
+ try {
+ ...
+ tm.commit(tx);
+ } catch (RollbackException e) {
+
+ // Here the transaction was rolled-back when
+ // trying to commit due to conflicts with other
+ // some other concurrent transaction
+ //
+ // The client application should do whatever is
+ // required according to its business logic
+ ...
+ }
+
+```
+
+A transaction can also be specifically aborted, for example if something goes wrong when executing the business logic:
+
+```java
+
+ ...
+ try {
+ ...
+ if( ! some_business_logic_condition ) {
+ tm.rollback(tx);
+ throw AnyApplicationException("Changes aborted due to...");
+ }
+ tm.commit(tx);
+ } catch (RollbackException e) {
+ ...
+ }
+
+```
+
+## Complete Example
+
+The following example summarizes the steps described above.
+
+```java
+
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import com.yahoo.omid.transaction.HBaseTransactionManager;
+import com.yahoo.omid.transaction.TTable;
+import com.yahoo.omid.transaction.Transaction;
+import com.yahoo.omid.transaction.TransactionManager;
+
+public class Example {
+
+ public static final byte[] exampleRow = Bytes.toBytes("EXAMPLE_ROW");
+ public static final byte[] family = Bytes.toBytes("EXAMPLE_CF");
+ public static final byte[] qualifier = Bytes.toBytes("foo");
+ public static final byte[] dataValueToAvoid = Bytes.toBytes("valToAvoid");
+ public static final byte[] dataValue = Bytes.toBytes("val");
+
+ public static void main(String[] args) throws Exception {
+
+ try (TransactionManager tm = HBaseTransactionManager.newInstance();
+ TTable txTable = new TTable("EXAMPLE_TABLE")) {
+
+ Transaction tx = tm.begin();
+ System.out.printl("Transaction started");
+
+ // Retrieve data transactionally
+ Get get = new Get(exampleRow);
+ get.add(family, qualifier);
+ Result txGetResult = txTable.get(tx, get);
+ byte[] retrievedValue = txGetResult.getValue(family, qualifier);
+
+ // Add a condition in the application logic to show explicit transaction
+ // aborts just for illustrative purposes
+ if (Bytes.equals(retrievedValue, dataValueToAvoid)) {
+ tm.rollback(tx);
+ throw new RuntimeException("Illegal value found in database!");
+ }
+
+ // Otherwise, add a value in other column and try to commit the transaction
+ try {
+ Put putOnRow = new Put(exampleRow);
+ putOnRow.add(family, qualifier, dataValue);
+ txTable.put(tx, putOnRow);
+ tm.commit(tx);
+ System.out.println("Transaction committed. New value written to example row");
+ } catch(RollbackException e) {
+ System.out.println("Transaction aborted due to conflicts. Changes to row aborted");
+ }
+
+ }
+
+ }
+
+}
+
+```
+
+## Additional Examples
+
+The `examples` module contains [complete examples](https://github.com/yahoo/omid/tree/master/examples/src/main/java/com/yahoo/omid/examples)
+showing the Omid functionality that can be executed in your Omid+HBase environment. Just clone the Omid project, go
+to the `examples` module and execute `mvn clean package` to create a tar.gz file that includes all the examples.
+In order to execute each example, just execute the `run.sh` script and follow the instructions.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/client-failure-management.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/client-failure-management.md b/doc/site/markdown/client-failure-management.md
new file mode 100644
index 0000000..9ca984e
--- /dev/null
+++ b/doc/site/markdown/client-failure-management.md
@@ -0,0 +1,30 @@
+# Management of Client Failures
+
+Upon user application crashes, Transactional Clients may leave some orphaned data in the datastore, even though
+the data in the datastore is kept in a consistent state. When another Transactional Client comes across this data,
+it will check whether the associated transaction was committed or not. If the data belongs to a transaction which
+has not been committed, and the transaction id is lower than the low watermark, then the data is deleted, since it
+belongs to a transaction that can never be committed.
+
+If data belongs to a transaction that was already committed, this means that the Transactional Client crashed between
+sending the commit request and completing the transaction in the commit table. This means that shadow cells have not
+been deleted. The Transactional Client reading will write the shadow cell for the orphan data. However, it is not able
+to complete the transaction in the commit table as it can not guarantee that shadow cells have been written for all cells
+in the transaction, since only the crashed client knew which cells it had written to. Consequently, this means that the
+commit table can grow indefinitely. Initially, this flaw should not cause many problems, since:
+
+* the amount of data stored is small (16 bytes)
+* the number of clients expected to fail between commit and completion is low
+* the data is stored on persistent storage. It is not bounded by memory.
+
+For example, if 1% of all Transactional Clients are expected to crash at this exact point in time, while the system is
+writing, on average, 100,000 transactions per second, the amount of data required to store these commit table entries
+for a whole year would be around 500 GB. A single disk could hold this.
+
+In any case, it would be desirable a mechanism to sanitize the Commit Table. If we can guarantee that all orphan data
+has been cleaned-up up to a certain low watermark, then all commit table entries whose start timestamp is lower than
+this low watermark can be deleted.
+
+In the case of HBase, there is already have a compactor which proactively cleans up transactions older than the low
+watermark. This could easily be extended to store the low watermark for each region it compacts, so then another process
+could clean up the commit table based on the minimum low watermark that was stored.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/coding-guide-and-style.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/coding-guide-and-style.md b/doc/site/markdown/coding-guide-and-style.md
new file mode 100644
index 0000000..504094f
--- /dev/null
+++ b/doc/site/markdown/coding-guide-and-style.md
@@ -0,0 +1,56 @@
+# Coding Guide
+
+The basic principle is to always write code that is testable, easy to understand, extensible, resistant to bugs. Code is written once but read by many people so we need to focus also on the next person that will read it:
+
+* Keep it simple
+* Test your code
+* [Single responsibility principle](http://programmer.97things.oreilly.com/wiki/index.php/The_Single_Responsibility_Principle)
+* Try to maintain components [loosely coupled](http://programmer.97things.oreilly.com/wiki/index.php/Cohesion_and_Coupling_matter)
+* Do not abuse of comments:
+ * Choose names that reveal intent for modules, packages, classes, methods, variables, constants, etc.
+ * Do not use comments for excuse bad code or when appropriate naming is enough
+ * Comments are good when they provide context (Explain why, not how) or explain things that happen in the real world
+* Follow the [Boy Scout Rule](http://programmer.97things.oreilly.com/wiki/index.php/The_Boy_Scout_Rule)
+* Don't reinvent the wheel:
+ * Use patterns when possible
+ * Use proven libraries for (e.g. Apache commons, guava, etc.)
+* Refactor when necessary
+ * When adding a new feature and the context is not appropriate, refactor first in a separate commit/s
+
+# Coding Style
+Omid coding style shoud follow [Oracle's Code Conventions for Java](http://www.oracle.com/technetwork/java/javase/documentation/codeconvtoc-136057.html), with the following modifications:
+
+* Lines can be up to 120 characters long
+* Indentation should be:
+ * 4 spaces
+ * Tabs not allowed
+* Always use curly braces for code blocks, even for single-line 'ifs' and 'elses'
+* Do not include @author tags in any javadoc
+* Use TestNG for testing
+* Import ordering and spacing:
+ * Try to organize imports alphabetically in blocks with this format:
+ * A first block of imports from external libraries
+ * A second block of imports from Java libraries
+ * Finally a third block with `static` imports
+ * Example:
+
+```java
+ import com.google.common.base.Charsets;
+ import com.yahoo.omid.zk.ZKUtils.ZKException;
+ import com.yahoo.statemachine.StateMachine.Event;
+ import com.yahoo.statemachine.StateMachine.Fsm;
+ import com.yahoo.statemachine.StateMachine.State;
+ import org.apache.commons.configuration.Configuration;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+ ...
+
+ import java.io.IOException;
+ import java.net.InetSocketAddress;
+ import java.util.ArrayDeque;
+ ...
+
+ import static com.yahoo.omid.ZKConstants.CURRENT_TSO_PATH;
+ import static com.yahoo.omid.zk.ZKUtils.provideZookeeperClient;
+ ...
+```
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/index.md b/doc/site/markdown/index.md
new file mode 100644
index 0000000..207d18c
--- /dev/null
+++ b/doc/site/markdown/index.md
@@ -0,0 +1,94 @@
+# What is Omid?
+
+**Apache Omid (Optimistically transaction Management In Datastores)** is a flexible, reliable, high performant
+and scalable transactional framework that allows Big Data applications to execute ACID transactions on top of
+MVCC key/value NoSQL datastores.
+
+The current implementation provides multi-row transactions on top of Apache HBase, but Omid's design is
+flexible enough to support other datastore implementations as long as they provide MVCC features in their API.
+
+The following sections introduce the motivation behind Omid and its high-level architecture and
+basic concepts. If you want to jump to a more hands-on approach to Omid, please jump to the [[Quickstart]] section.
+On the other hand, if you want more information about Omid's design and its internals, please refer to the
+sections in the Technical Documentation menuf.
+
+# Why Omid?
+
+A *transaction* comprises a set of data manipulation operations on the state of a database system managed as a single
+unit of work, so all the operations must either entirely be completed (committed) or have no effect (aborted). In other
+words, partial executions of the transaction are not permitted (nor desired in general) because the final state of the
+database can be corrupted.
+
+Without the support for transactions, developers are burdened with ensuring atomic execution of scattered changes in
+data upon failures as well as when there are concurrent accesses to the same data by multiple clients.
+
+In order to process transactions, database systems provide a specific component called the *Transaction Manager*.
+The goal of transaction managers in general is to guarantee the so-called **ACID properties** of transactions:
+*Atomicity*, *Consistency*, *Isolation* and *Durability*. However, ACID properties are hard to scale when databases
+have to deal with very large amounts of data and thousands of concurrent users, because the data must be partitioned,
+distributed and replicated. That was the main reason why, with the advent of NoSQL big data stores, transactions were
+initially left out of the equation. HBase, Dynamo, BigTable, PNUTS, Cassandra, etc. lacked this precious feature
+initially. However, with the popularization of NoSQL big datastores in many areas of the industry, the need for
+transactions has become a must for certain applications.
+
+Omid fills this gap and provides lock-free transactional support on top of HBase with snapshot isolation guarantees.
+Omid enables BigData applications to benefit from the best of both worlds: the scalability provided by NoSQL
+datastores such as HBase, and the concurrency and atomicity provided by transaction processing systems.
+
+## Benefits
+
+The benefits that Omid provides include:
+
+1. Omid allows multi-row/multi-table transactions on top of HBase.
+2. Omid is lock-free. In lock-based approaches, the locks on data that are held by the incomplete transactions of
+a failed client prevent others from progressing. In Omid, if a client is slow or faulty, it does not slow down the
+other clients.
+3. Omid provides [Snapshot Isolation](basic-concepts.html#Snapshot_Isolation) (SI) guarantees.
+4. Omid does not require any modification into HBase code. All the transactional logic is implemented in the framework
+components.
+5. Omid does not require any change into HBase table schema. Omid uses the HBase metadata -the cell timestamp in
+particular- to store the transaction timestamp of each value inserted, updated or deleted in a cell. This enables
+concurrent transactions to read data from the right snapshot.
+6. Omid is being used internally at Yahoo in production systems, exhibiting good performance and reliability.
+
+# Architecture Overview
+
+The main architectural components are represented in the figure below and described briefly below in the following
+paragraphs.
+
+
+
+For a detailed picture of the Omid architecture, please refer to the [[Omid Components]] section in the Technical Documentation.
+
+## Component Roles
+
+Omid benefits from a centralized scheme in which a single server, called Transactional Status Oracle (TSO), monitors the
+modified rows/cells by transactions and use that to detect write-write conflicts.
+
+User applications are allowed to begin, commit or abort transactions by means of Transactional Clients (TC), which
+enable remote communication to the TSO and allow to perform transactional operations on the data stored in the datastore.
+
+When a transaction is created, a unique start timestamp is assigned by the Timestamp Oracle (TO). This start timestamp
+serves as a transaction identifier and is used by the TSO to guarantee SI by detecting conflicts in the writesset of
+a committing transaction with other concurrent transactions. Upon, finishing conflict detection successfully, the
+TSO assigns the transaction a commit timestamp and writes the mapping start/commit timestamp in the Commit Table (CT)
+before returning the response to the client. When receiving the response. the transactional client, adds a Shadow
+Cell (SC) per cell in the transaction writeset in order to allow to resolve the right snapshot for further read
+operations without disturbing the TSO.
+
+The main functionality provided by each component depicted in the figure above can be summarized as follows:
+
+* **Timestamp Oracle (TO)** It assigns the _start_ and _commit timestamps_ that demarcate transactional contexts.
+* **Transaction Status Oracle (TSO)**. Its the central element of the Omid architecture on the server-side. Its main
+task is to resolve conflicts between concurrent transactions.
+* **Commit Table (CT)** It stores a temporary mapping from the start timestamp to the commit timestamp of every
+committed transaction.
+* **Transactional Client (TC)** It allows client applications to demarcate transaction boundaries through the so-called
+_Transaction Manager_ and to perform transactional read/write operations on data through _Transactional Tables_.
+* **Shadow Cells (SC)** These special metadata cells are written alongside data cells in the data store to allow the
+transactional clients to resolve read snapshots without consulting the Commit Table. They contain the transactional
+boundaries of the last transaction in writing to the data cell.
+
+For a more in-depth description of how Omid works, please refer to the sections in the Technical Documentation menu.
+
+**Do you want to try Omid now?** Please, go to the [[Quickstart]] section.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/mailing-lists.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/mailing-lists.md b/doc/site/markdown/mailing-lists.md
new file mode 100644
index 0000000..adddf1d
--- /dev/null
+++ b/doc/site/markdown/mailing-lists.md
@@ -0,0 +1,13 @@
+# Mailing Lists @ Apache Incubator
+
+| Developers | Commits |
+| -------------------------------- | --------------------------------------|
+| dev@omid.incubator.apache.org | commits@omid.incubator.apache.org |
+
+## Subscribe
+
+In order to subscribe to a list, please add the '-subscribe' suffix to the identifier in each list and send an email.
+For example use dev-subscribe@omid.incubator.apache.org to subscribe to the list of developers.
+
+## Unsubscribe
+Use the '-unsubscribe' suffix to unsubscribe from a list.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/omid-components.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/omid-components.md b/doc/site/markdown/omid-components.md
new file mode 100644
index 0000000..2739a23
--- /dev/null
+++ b/doc/site/markdown/omid-components.md
@@ -0,0 +1,96 @@
+# Omid Architecture and Component Description
+
+The following figure depicts the current Omid architecture presented in the section presenting Omid:
+
+
+
+The following sections describe more in deep the most important components and elements in the Omid architecture.
+
+## Transactional Clients (TCs)
+**Transactional Clients (TCs)** are the entry point to user applications using transactions with Omid. They allow
+applications to *demarcate transactional boundaries* and *read/write transactionally from/to the data source*,
+HBase in this case.
+
+There are two main interfaces/classes that compose the Transactional Client:
+
+1. The Transaction Manager interface (represented by the `TransactionManager` interface in the current implementation):
+Allows user applications to demarcate transactional contexts, providing methods to begin(), commit(), rollback()
+and markRollbackOnly() transactions.
+2. The Data Operation interface (represented by the `TTable` class in the HBase implementation): Allows user applications
+to trigger transactional operations to the datasource. For example in the HBase, put, get and scan operations.
+
+Examples about how to add transactions using these interfaces are described in the [[Basic Examples]] section.
+
+## Timestamp Oracle (TO)
+The single responsibility of the Timestamp Oracle is to manage transaction timestamps. Transaction timestamps serve as a
+logical clock and are used mainly to preserve the time-related guarantees required for Snapshot Isolation.
+
+The TO allocates and delivers transaction timestamps when required by the TSO. To achieve this task, it maintains a
+monotonically increasing counter. Transaction timestamps are allocated when a transaction begins and right after a transaction
+willing to commit has passed the writeset validation. The start timestamp is also used as transaction identifier.
+
+In order to guarantee that all timestamps are unique, the TO has to persist a value after every N timestamp allocations. This
+value will be used for recovering the TSO in case of crashes. The value persisted is the current timestamp value plus N, N being
+a congurable property. From that point on, the TO is then free to safely allocate timestamps up to that value.
+
+In the current implementation, the maximum allocated timestamp value can be persisted/stored in:
+
+* ZooKeeper
+* In its own HBase table
+
+## The Server Oracle (TSO)
+The TSO is the core component in Omid. There is a single instance of the TSO (Although a high availability protocol is currently
+a WIP). Its unique responsibility is to resolve conflicts between concurrent transactions. The TSO must maintain the last allocated
+timestamp, the so-called low watermark and a conflict map for detecting conflicts in the writesets of concurrent transactions. It
+also delegates the update and persistence of a list of committed transactions to the commit table component (see Figure). Aborted and
+uncommitted transactions are not tracked. Maintaining the TSO state to the minimum allows us to avoid contention in many cases and
+provides fast recovery upon crashes.
+
+As mentioned before, conflict resolution for transactions is the primary function of the TSO. To achieve this task, it uses a large
+map structure of (cellId -> lastCommittedTransaction) pairs called the conflict map. When the client commits a transaction, it sends
+the cell ids of the cells being committed, i.e. its writeset, to the TSO, which checks possible conflicts when other transactions
+that were concurrent using the conflict map. If no conflicts are found, a commit timestamp is allocated and the lastCommittedTransaction
+for each cell id in the transaction is updated to this timestamp.
+
+Confict detection is implemented by comparing the hash of the cell ids in the writeset, not the actual cell id value. As
+the conflict map does not have a one to one mapping with each cell in cell in the database, this means that there's a possibility of
+false aborts in the system. The probability of false aborts is a variant of the [birthday problem](http://en.wikipedia.org/wiki/Birthday_problem)
+and basically translates into the fact that long-running transactions are more likely to have false aborts.
+
+The low watermark is only updated when a value is replaced in the conflict map. The current implementation of the conflict map uses an
+open addressing type mechanism, using linear probing with a limit to the amount of probing. When a cell is checked it is hashed to a
+location in the map. If the location occupied, the location is checked and so on. If the cell finds an entry which matches its cell id,
+it replaces it. Otherwise, if the process gets to the end of the probe without finding an empty location, it replaces the cell with
+the oldest commit timestamp. The oldest commit timestamp then becomes the low watermark in the TSO. This must occur to ensure that
+conflicts are not missed, as it would be possible for a row to not find a potential conflict that had been overridden if the low watermark
+was not updated.
+
+### Processing Commit Requests
+When processing the commit request for a transaction, the TSO first checks if the start timestamp is greater the low watermark and if so,
+if there was another concurrent transaction whose writeset overlapped and whose commit timestamp is greater the start timestamp of
+the requesting transaction. If no conflicts are found, a commit timestamp is generated for the request transaction which is added
+to the conflict map for the writeset, the commit is added to the commit table and finally the client is notified about the successful
+commit. Otherwise, the commit fails and the client is notied on the rollback of the transaction.
+
+## Commit Table (CT)
+It contains a transient mapping from start timestamp to commit timestamp for recently committed transactions. The commit table is
+stored on a different set of nodes to the TSO. When a transaction is committed on the TSO it must be stored in the commit table
+before a response is sent to the client. As the key value mapping in the commit table are completely independent of each other, the
+table can be trivially scaled by sharding the values over multiple nodes.
+
+The commit table provides two services to the Transactional Client. Firstly, it is used to check if a transaction has been committed
+or not. The client should not need to consult the commit table in the common case as the status of a read cell should be available
+from its shadow cell (see below).
+
+The other service provided by the commit table is that transaction clients can delete commits from it. This only occurs once the writing
+client has updated shadow cells for each of the cells written in the transaction. We call this commit or transaction completion. It is
+possible that a client will crash between a transaction being committed and being completed. In this case, the transaction will stay in
+the commit table forever. However as the commit table is not stored in memory and the window of opportunity for such a crash is quite
+low, we don't expect this to be a problem.
+
+## Shadow Cells (SCs)
+Shadow cells contain a mapping from start timestamp to commit timestamp of the last committed transaction that modified the cell. Upon
+receiving a commit timestamp after a successful transaction commit, the Transactional Client adds/updates these special cells to each
+modified cell in the writeset of the committed transaction. The purpose of this, is that Transactional Clients can then query the
+corresponding shadow cell when they read each cell to check if the data they find in the cell has been committed or not and whether
+it is within the snapshot of the reading transaction.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/markdown/quickstart.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/quickstart.md b/doc/site/markdown/quickstart.md
new file mode 100644
index 0000000..5976db8
--- /dev/null
+++ b/doc/site/markdown/quickstart.md
@@ -0,0 +1,203 @@
+# Quickstart
+
+Below are instructions to quickly set up an environment to test Omid in your local machine.
+
+## Requirements
+
+1. Maven 3.x
+2. Java 7
+3. HBase 0.98
+4. Protobuf 2.5.0
+
+## TSO Setup
+
+### 1. Download and Install the Required HBase Version
+You can find HBase distributions in [this page](http://www.apache.org/dyn/closer.cgi/hbase/).
+Then start HBase in [standalone mode](https://hbase.apache.org/book.html#quickstart).
+
+### 2. Clone the [Omid repository](https://github.com/francisco-perez-sorrosal/omid) and Build the TSO Package:
+
+```sh
+$ git clone git@github.com:yahoo/omid.git
+$ cd omid
+$ mvn clean install
+```
+This will generate a binary package containing all dependencies for the TSO in tso-server/target/tso-server-\<VERSION\>-bin.tar.gz.
+
+**Be aware** Unit tests use HBase mini cluster, it typically fails to start if you are on VPN, thus unit test fail.
+Unit tests coverage is also quite extensive and take a while to run on each build (~15min at the moment of writing). So, consider using
+`mvn clean install -DskipTests` to speed temporal builds. Note that `-Dmaven.test.skip=true` [is NOT an equivalent](http://ericlefevre.net/wordpress/2008/02/21/skipping-tests-with-maven/).
+
+As an alternative to clone the project, you can download the required version for the TSO tar.gz package from the [release repository](https://bintray.com/yahoo/maven/omid/view).
+
+You can also see the [build history here](https://github.com/yahoo/omid/tags).
+
+### 3. Extract the TSO Package
+
+```sh
+$ tar zxvf tso-server-<VERSION>-bin.tar.gz
+$ cd tso-server-<VERSION>
+```
+
+### 4. Create Omid Tables
+Ensure that the setting for hbase.zookeeper.quorum in conf/hbase-site.xml points to your zookeeper instance, and create the
+Timestamp Table and the Commit Table using the omid.sh script included in the bin directory of the tso server:
+
+```sh
+$ bin/omid.sh create-hbase-commit-table -numRegions 16
+$ bin/omid.sh create-hbase-timestamp-table
+```
+
+These two tables are required by Omid and they must not be accessed by client applications.
+
+### 5. Start the TSO Server
+
+```sh
+$ bin/omid.sh tso
+```
+
+This starts the TSO server that in turn will connect to HBase to store information in HBase. By default the TSO listens on
+port 54758. If you want to change the TSO configuration, you can modify the contents in the conf/omid.conf file.
+
+## HBase Client Usage
+
+### 1. Create a New Java Application
+Use your favorite IDE an create a new project.
+
+### 2. Add hbase-client Dependency
+Choose the right version of the hbase-client jar. For example, in a Maven-based app add the following dependency in the
+pom.xml file:
+
+```xml
+<dependency>
+ <groupId>com.yahoo.omid</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase_client.version}</version>
+</dependency>
+```
+
+### 3. Start Coding Using the Omid Client Interfaces
+In Omid there are two client interfaces: `TTable` and `TransactionManager` (_These interfaces will likely change slightly
+in future._):
+
+1. The `TransactionManager` is used for creating transactional contexts, that is, transactions. A builder is provided in
+the `HBaseTransactionManager` class in order to get the TransactionManager interface.
+
+2. `TTable` is used for putting, getting and scanning entries in a HBase table. TTable's
+interface is similar to the standard `HTableInterface`, and only requires passing the transactional context as a
+first parameter in the transactional aware methods (e.g. `put(Transaction tx, Put put)`)
+
+## Example Application
+
+Below is provided a sample application accessing data transactionally. Its a dummy application that writes two cells in two
+different rows of a table in a transactional context, but is enough to show how the different Omid client APIs are used. A
+detailed explanation of the client interfaces can be found in the [Basic Examples] section.
+
+```java
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import com.yahoo.omid.transaction.HBaseTransactionManager;
+import com.yahoo.omid.transaction.TTable;
+import com.yahoo.omid.transaction.Transaction;
+import com.yahoo.omid.transaction.TransactionManager;
+
+public class OmidExample {
+
+ public static final byte[] family = Bytes.toBytes("MY_CF");
+ public static final byte[] qualifier = Bytes.toBytes("MY_Q");
+
+ public static void main(String[] args) throws Exception {
+
+ try (TransactionManager tm = HBaseTransactionManager.newInstance();
+ TTable txTable = new TTable("MY_TX_TABLE")) {
+
+ Transaction tx = tm.begin();
+
+ Put row1 = new Put(Bytes.toBytes("EXAMPLE_ROW1"));
+ row1.add(family, qualifier, Bytes.toBytes("val1"));
+ txTable.put(tx, row1);
+
+ Put row2 = new Put(Bytes.toBytes("EXAMPLE_ROW2"));
+ row2.add(family, qualifier, Bytes.toBytes("val2"));
+ txTable.put(tx, row2);
+
+ tm.commit(tx);
+
+ }
+
+ }
+
+}
+```
+
+To run the application, make sure `core-site.xml` and `hbase-site.xml` for your HBase cluster are present in your
+CLASSPATH. The default configuration settings for the Omid client are loaded from the `default-hbase-omid-client-config.yml`
+file. In order to change the client default settings, you can either; 1) put your specific configuration settings
+in a file named `hbase-omid-client-config.yml` and include it in the CLASSPATH; or 2) do it programmatically in
+the application code by creating an instance of the `HBaseOmidClientConfiguration` class and passing it in the
+creation of the transaction manager:
+
+```java
+ import com.yahoo.omid.transaction.HBaseOmidClientConfiguration;
+
+ ...
+
+ HBaseOmidClientConfiguration omidClientConfiguration = new HBaseOmidClientConfiguration();
+ omidClientConfiguration.setConnectionType(DIRECT);
+ omidClientConfiguration.setConnectionString("my_tso_server_host:54758");
+ omidClientConfiguration.setRetryDelayMs(3000);
+
+ try (TransactionManager tm = HBaseTransactionManager.newInstance(omidClientConfiguration);
+ TTable txTable = new TTable("MY_TX_TABLE")) {
+
+ ...
+```
+
+Also, you will need to create a HBase table "MY_TX_TABLE", with column family "MY_CF", and with `TTL` disabled and
+`VERSIONS` set to `Integer.MAX_VALUE`. For example using the HBase shell:
+
+```
+create 'MY_TX_TABLE', {NAME => 'MY_CF', VERSIONS => '2147483647', TTL => '2147483647'}
+```
+
+This example assumes non-secure communication with HBase. If your HBase cluster is secured with Kerberos, you will need to
+use the `UserGroupInformation` API to log in securely.
+
+## The Omid Compactor Coprocessor
+
+Omid includes a jar with an HBase coprocessor for performing data cleanup that operates during compactions, both minor and
+major. Specifically, it does the following:
+
+* Cleans up garbage data from aborted transactions
+* Purges deleted cells. Omid deletes work by placing a special tombstone marker in cells. The compactor
+ detects these and actually purges data when it is safe to do so (i.e. when there are no committable transactions
+ that may read the data).
+* 'Heals' committed cells for which the writer failed to write shadow cells.
+
+To deploy the coprocessor, the coprocessor jar must be placed in a location (typically on HDFS) that is accessible
+by HBase region servers. The coprocessor may then be enabled on a transactional table by the following steps in the HBase shell:
+
+**1) Disable the table**
+
+```
+disable 'MY_TX_TABLE'
+```
+
+**2) Add a coprocessor specification to the table via a "coprocessor" attribute. The coprocessor spec may (and usually will)
+also include the name of the Omid commit table**
+
+```
+alter 'MY_TX_TABLE', METHOD => 'table_att', 'coprocessor'=>'<path_to_omid_coprocessor>/omid-hbase-coprocessor-<coprocessor_version>.jar|com.yahoo.omid.transaction.OmidCompactor|1001|omid.committable.tablename=OMID_COMMIT_TABLE'
+```
+
+**3) Add an "OMID_ENABLED => true" flag to any column families which the co-processor should work on**
+
+```
+alter 'MY_TX_TABLE', { NAME => 'MY_CF', METADATA => {'OMID_ENABLED' => 'true'}}
+```
+
+**4) Re-enable the table**
+
+```
+enable 'MY_TX_TABLE'
+```
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/css/site.css
----------------------------------------------------------------------
diff --git a/doc/site/resources/css/site.css b/doc/site/resources/css/site.css
new file mode 100644
index 0000000..7b37a01
--- /dev/null
+++ b/doc/site/resources/css/site.css
@@ -0,0 +1,103 @@
+/* Basics */
+
+a {
+ color: #B9140A;
+}
+
+body {
+ font-size: 16px;
+ font-family: 'Ubuntu', sans-serif;
+}
+
+/* Navigation Bar */
+
+.navbar .container {
+ color: white;
+ background-color: #B9140A;
+ background-image: none;
+}
+
+.navbar .nav>li>a {
+ color: white;
+ text-shadow: 0px 0px 10px black;
+}
+
+.navbar .nav li.dropdown.open>.dropdown-toggle {
+ color: #08C;
+ text-shadow: none;
+}
+
+.navbar .nav>li>a:focus, .navbar .nav>li>a:hover, .navbar .nav>li>a:active {
+ color: white;
+ text-shadow: 0px 0px 20px white;
+}
+
+
+/* Tables */
+
+table td, table th {
+ border: 1px solid #333;
+ padding: 0 .5em;
+}
+
+table th {
+ background: #CCC;
+ padding: 0 .5em;
+}
+
+table {
+ margin-top: 20px;
+ margin-bottom: 20px;
+}
+
+
+/* Footer */
+
+footer {
+ color: white;
+ text-shadow: 0px 0px 10px black;
+ background-color: #B9140A;
+}
+
+footer .container {
+ background-color: #B9140A;
+ background-image: none;
+}
+
+footer .container a {
+ color: white;
+ text-shadow: 0px 0px 10px black;
+ text-decoration: underline;
+}
+
+footer .pull-right {
+ color: white;
+ text-shadow: 0px 0px 10px black;
+}
+
+/* Others */
+
+.row {
+ margin-top: 20px;
+}
+
+.breadcrumb {
+ padding: 20px 15px;
+ background-color: #B9140A;
+}
+
+
+#search-form {
+ margin-left: 9px;
+ margin-right: 40px;
+}
+
+#publishDate {
+ color: white;
+ text-shadow: 0px 0px 10px black;
+}
+
+#projectVersion {
+ color: white;
+ text-shadow: 0px 0px 10px black;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/about-omid.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/about-omid.png b/doc/site/resources/images/about-omid.png
new file mode 100644
index 0000000..2875c7c
Binary files /dev/null and b/doc/site/resources/images/about-omid.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/architecture.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/architecture.png b/doc/site/resources/images/architecture.png
new file mode 100644
index 0000000..9afe38b
Binary files /dev/null and b/doc/site/resources/images/architecture.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/basic-alg.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/basic-alg.png b/doc/site/resources/images/basic-alg.png
new file mode 100644
index 0000000..82054f0
Binary files /dev/null and b/doc/site/resources/images/basic-alg.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/contact.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/contact.png b/doc/site/resources/images/contact.png
new file mode 100644
index 0000000..e1329e8
Binary files /dev/null and b/doc/site/resources/images/contact.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/getting-started.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/getting-started.png b/doc/site/resources/images/getting-started.png
new file mode 100644
index 0000000..c7ac001
Binary files /dev/null and b/doc/site/resources/images/getting-started.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/omid-logo-transparent.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/omid-logo-transparent.png b/doc/site/resources/images/omid-logo-transparent.png
new file mode 100644
index 0000000..842f30b
Binary files /dev/null and b/doc/site/resources/images/omid-logo-transparent.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/omid-logo.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/omid-logo.png b/doc/site/resources/images/omid-logo.png
new file mode 100644
index 0000000..ccce0ef
Binary files /dev/null and b/doc/site/resources/images/omid-logo.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/snapshot-isolation.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/snapshot-isolation.png b/doc/site/resources/images/snapshot-isolation.png
new file mode 100644
index 0000000..7ede9ee
Binary files /dev/null and b/doc/site/resources/images/snapshot-isolation.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/resources/images/technical-documentation.png
----------------------------------------------------------------------
diff --git a/doc/site/resources/images/technical-documentation.png b/doc/site/resources/images/technical-documentation.png
new file mode 100644
index 0000000..23c1f7a
Binary files /dev/null and b/doc/site/resources/images/technical-documentation.png differ
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/doc/site/site.xml
----------------------------------------------------------------------
diff --git a/doc/site/site.xml b/doc/site/site.xml
new file mode 100644
index 0000000..0afdb73
--- /dev/null
+++ b/doc/site/site.xml
@@ -0,0 +1,85 @@
+<project name="Apache Omid">
+
+ <skin>
+ <groupId>org.apache.maven.skins</groupId>
+ <artifactId>maven-fluido-skin</artifactId>
+ <version>1.5</version>
+ </skin>
+
+ <custom>
+ <fluidoSkin>
+
+ <topBarEnabled>true</topBarEnabled>
+ <topBarContainerStyle>width: 100%;</topBarContainerStyle>
+ <sideBarEnabled>false</sideBarEnabled>
+
+ <twitter>
+ <user>apacheomid</user>
+ <showUser>true</showUser>
+ <showFollowers>true</showFollowers>
+ </twitter>
+
+ <googleSearch/>
+
+ </fluidoSkin>
+ </custom>
+
+ <bannerLeft>
+ <name>Omid</name>
+ <src>images/omid-logo.png</src>
+ <href>http://omid.incubator.apache.org</href>
+ <width>200</width>
+ </bannerLeft>
+
+ <bannerRight>
+ <name>Apache Incubator</name>
+ <src>http://incubator.apache.org/images/egg-logo2.png</src>
+ <href>http://incubator.apache.org/</href>
+ <width>200</width>
+ </bannerRight>
+
+ <publishDate position="right"/>
+ <!-- Do not publish version as it shows the current SNAPSHOT -->
+ <!-- version position="right"/ -->
+
+ <poweredBy>
+ <logo name="Maven" href="http://maven.apache.org/" img="http://maven.apache.org/images/logos/maven-feather.png"/>
+ </poweredBy>
+
+ <body>
+
+ <menu name="Home">
+ <item name="Overview" href="index.html" />
+ <item name="License" href="license.html" />
+ </menu>
+
+ <menu name="Download">
+ <item name="Omid Sources" href="https://github.com/yahoo/omid" />
+ <item name="Bintray Repository" href="https://bintray.com/yahoo/maven/omid/_latestVersion" />
+ </menu>
+
+ <menu name="User Guide & API">
+ <item name="Quickstart" href="quickstart.html" />
+ <item name="API and Code Examples" href="basic-examples.html" />
+ </menu>
+
+ <menu name="Technical Docs">
+ <item name="Basic Concepts" href="basic-concepts.html" />
+ <item name="Omid Components" href="omid-components.html" />
+ <item name="Basic Algorithm" href="basic-algorithm.html" />
+ <item name="Management of Client Failures" href="client-failure-management.html" />
+ <item name="Blog Entries" href="http://yahoohadoop.tumblr.com/tagged/HBase" />
+ </menu>
+
+ <menu name="Contribute">
+ <item name="Source Code" href="https://git-wip-us.apache.org/repos/asf/incubator-omid.git" />
+ <item name="JIRA" href="https://issues.apache.org/jira/browse/Omid" />
+ <item name="Mailing Lists" href="mailing-lists.html" />
+ <item name="Coding Guide and Style" href="coding-guide-and-style.html" />
+ </menu>
+
+ <menu name="Project Reports" ref="reports" />
+
+ </body>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 192f414..a39d652 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -99,6 +99,14 @@
</configuration>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 1b2e25c..a0869a5 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -145,6 +145,13 @@
</configuration>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index 82dc0f7..c623ebf 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -120,6 +120,22 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index d11e642..d912e4c 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -61,6 +61,22 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index ffc16c5..beec0b4 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -130,6 +130,14 @@
</configuration>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index 32576f2..c8eddd1 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -64,4 +64,20 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
</project>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index a43fadc..c28c507 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -77,6 +77,22 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index 5e1b032..e4dd5e0 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -51,4 +51,20 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 27d1268..823db65 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,15 +6,20 @@
<!-- Project description -->
<!-- ========================================================================================================== -->
<name>Omid</name>
- <description>The Omid project provides transactional support for HBase-based applications</description>
+ <description>Apache Omid provides multi-row/multi-table transactional support for HBase-based applications</description>
<inceptionYear>2011</inceptionYear>
- <url>http://www.github.org/apache/omid</url>
+ <url>http://omid.incubator.apache.org</url>
<groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
<version>0.8.1.37-SNAPSHOT</version>
+ <organization>
+ <name>Apache Software Foundation</name>
+ <url>http://www.apache.org</url>
+ </organization>
+
<licenses>
<license>
<name>Apache License, Version 2.0</name>
@@ -23,6 +28,51 @@
</license>
</licenses>
+ <developers>
+
+ <developer>
+ <id>daijy</id>
+ <name>Daniel Dai</name>
+ <email>daijy@apache.org</email>
+ <timezone>-8</timezone>
+ <organization>Hortonworks</organization>
+ <organizationUrl>http://www.hortonworks.com</organizationUrl>
+ </developer>
+ <developer>
+ <id>fperezsorrosal</id>
+ <name>Francisco Perez-Sorrosal</name>
+ <email>fperezsorrosal@apache.org</email>
+ <timezone>-8</timezone>
+ <organization>Yahoo Inc.</organization>
+ <organizationUrl>http://www.yahoo.com</organizationUrl>
+ </developer>
+ <developer>
+ <id>ikatkov</id>
+ <name>Igor Katkov</name>
+ <email>ikatkov@apache.org</email>
+ <timezone>-8</timezone>
+ <organization>Yahoo Inc.</organization>
+ <organizationUrl>http://www.yahoo.com</organizationUrl>
+ </developer>
+ <developer>
+ <id>ohads</id>
+ <name>Ohad Shacham</name>
+ <email>ohads@apache.org</email>
+ <timezone>+2</timezone>
+ <organization>Yahoo Inc.</organization>
+ <organizationUrl>http://www.yahoo.com</organizationUrl>
+ </developer>
+ <developer>
+ <id>sameerp</id>
+ <name>Sameer Paranjpye</name>
+ <email>sameerp@apache.org</email>
+ <timezone>-8</timezone>
+ <organization>Arimo</organization>
+ <organizationUrl>https://www.arimo.com</organizationUrl>
+ </developer>
+
+ </developers>
+
<modules>
<module>benchmarks</module>
<module>common</module>
@@ -42,11 +92,20 @@
<module>examples</module>
</modules>
+ <pluginRepositories>
+
+ <pluginRepository>
+ <id>synergian-repo</id>
+ <url>https://raw.github.com/synergian/wagon-git/releases</url>
+ </pluginRepository>
+
+ </pluginRepositories>
+
<scm>
- <connection>scm:git:git://github.org/apache/omid.git</connection>
- <developerConnection>scm:git:https://${GH_TOKEN}@github.org/apache/omid.git</developerConnection>
- <url>https://github.org/apache/omid</url>
- <tag>omid-0.8.1.21</tag>
+ <connection>scm:git:git://github.com/yahoo/omid.git</connection>
+ <developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
+ <url>https://github.com/yahoo/omid</url>
+ <tag>master</tag>
</scm>
<distributionManagement>
@@ -54,6 +113,15 @@
<id>bintray</id>
<url>https://api.bintray.com/maven/yahoo/maven/omid</url>
</repository>
+
+ <site>
+ <id>apache-omid-site</id>
+ <!-- Wagon-git URL format:
+ protocol:branch://url
+ Do not forget the 2 backslashes before the url
+ -->
+ <url>git:asf-site://https://git-wip-us.apache.org/repos/asf/incubator-omid-site.git</url>
+ </site>
</distributionManagement>
<properties>
@@ -102,11 +170,11 @@
<maven-coveralls-plugin.version>4.1.0</maven-coveralls-plugin.version>
<maven-cobertura-plugin.version>2.7</maven-cobertura-plugin.version>
<maven-license-plugin.version>2.11</maven-license-plugin.version>
+ <maven-site-plugin.version>3.5</maven-site-plugin.version>
<!-- Licensing properties (for license-maven-plugins) -->
<license.header>misc/header.txt</license.header>
-
</properties>
<build>
@@ -195,7 +263,6 @@
</configuration>
</plugin>
-
<!-- ============================================================================================== -->
<!-- Code coverage plugins -->
<!-- ============================================================================================== -->
@@ -337,8 +404,36 @@
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>${maven-site-plugin.version}</version>
+ <configuration>
+ <siteDirectory>${basedir}/doc/site</siteDirectory>
+ <outputDirectory>${basedir}/generated-website</outputDirectory>
+ <stagingDirectory>${basedir}/generated-website-staging</stagingDirectory>
+ </configuration>
+ <dependencies>
+ <dependency><!-- add support for ssh/scp -->
+ <groupId>org.apache.maven.wagon</groupId>
+ <artifactId>wagon-ssh</artifactId>
+ <version>1.0</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
</plugins>
+ <extensions>
+
+ <extension>
+ <groupId>ar.com.synergian</groupId>
+ <artifactId>wagon-git</artifactId>
+ <version>0.2.5</version>
+ </extension>
+
+ </extensions>
+
</build>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index 660770e..b240062 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -37,4 +37,20 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
</project>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index 3d379ad..2120458 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -107,6 +107,22 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index e0102a5..e5a0d1e 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -98,4 +98,20 @@
</dependencies>
+ <build>
+
+ <plugins>
+
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
</project>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/05d69d9a/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index 0cf0f6c..f60e2f2 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -255,6 +255,14 @@
</executions>
</plugin>
+ <!-- Disable maven site plugin -->
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
[18/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
deleted file mode 100644
index 71e2ecb..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
+++ /dev/null
@@ -1,418 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.base.Charsets;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.tso.TSOStateManager.TSOState;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.test.TestingServer;
-import org.apache.curator.utils.CloseableUtils;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-
-import static com.yahoo.omid.tso.client.TSOClient.DEFAULT_ZK_CLUSTER;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class TestLeaseManager {
-
- private static final long DUMMY_EPOCH_1 = 1L;
- private static final long DUMMY_EPOCH_2 = 2L;
- private static final long DUMMY_EPOCH_3 = 3L;
- private static final long DUMMY_LOW_WATERMARK_1 = DUMMY_EPOCH_1;
- private static final long DUMMY_LOW_WATERMARK_2 = DUMMY_EPOCH_2;
- private static final long DUMMY_LOW_WATERMARK_3 = DUMMY_EPOCH_3;
-
- private static final String LEASE_MGR_ID_1 = "LM1";
- private static final String LEASE_MGR_ID_2 = "LM2";
- private static final String INSTANCE_ID_1 = "LM1" + "#";
- private static final String INSTANCE_ID_2 = "LM2" + "#";
-
- private static final Logger LOG = LoggerFactory.getLogger(TestLeaseManager.class);
-
- private static final long TEST_LEASE_PERIOD_IN_MS = 1000; // 1 second
-
- private CuratorFramework zkClient;
- private TestingServer zkServer;
-
- @Mock
- private Panicker panicker;
-
- private PausableLeaseManager leaseManager1;
- private PausableLeaseManager leaseManager2;
-
- @BeforeClass
- public void beforeClass() throws Exception {
-
- LOG.info("Starting ZK Server");
- zkServer = TestUtils.provideTestingZKServer();
- LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
-
- zkClient = TestUtils.provideConnectedZKClient(DEFAULT_ZK_CLUSTER);
-
- }
-
- @AfterClass
- public void afterClass() throws Exception {
-
- zkClient.close();
-
- CloseableUtils.closeQuietly(zkServer);
- zkServer = null;
- LOG.info("ZK Server Stopped");
-
- }
-
- @Test(timeOut = 30_000)
- public void testErrorInitializingTSOStateExitsTheTSO() throws Exception {
-
- final String TEST_TSO_LEASE_PATH = "/test0_tsolease";
- final String TEST_CURRENT_TSO_PATH = "/test0_currenttso";
-
- Panicker panicker = spy(new MockPanicker());
-
- TSOChannelHandler tsoChannelHandler = mock(TSOChannelHandler.class);
- TSOStateManager stateManager = mock(TSOStateManager.class);
- when(stateManager.reset()).thenThrow(new IOException());
- leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
- tsoChannelHandler,
- stateManager,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
- leaseManager1.startService();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- verify(panicker, timeout(2000).atLeastOnce()).panic(anyString(), any(IOException.class));
-
- leaseManager1.stopService();
-
- }
-
- @Test(timeOut = 60000)
- public void testLeaseHolderDoesNotChangeWhenPausedForALongTimeAndTheresNoOtherInstance()
- throws Exception {
-
- final String TEST_TSO_LEASE_PATH = "/test1_tsolease";
- final String TEST_CURRENT_TSO_PATH = "/test1_currenttso";
-
- // Launch the instance under test...
- TSOChannelHandler tsoChannelHandler1 = mock(TSOChannelHandler.class);
- TSOStateManager stateManager1 = mock(TSOStateManager.class);
- when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
- leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
- tsoChannelHandler1,
- stateManager1,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
- leaseManager1.startService();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... check is the lease holder
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
- assertTrue(leaseManager1.stillInLeasePeriod());
-
- // Then, pause instance when trying to renew lease...
- leaseManager1.pausedInTryToRenewLeasePeriod();
-
- // ...let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ...check that nothing changed...
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
-
- // Finally, resume the instance...
- leaseManager1.resume();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... and check again that nothing changed
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
- assertTrue(leaseManager1.stillInLeasePeriod());
-
- }
-
- @Test(timeOut = 60_000)
- public void testLeaseHolderDoesNotChangeWhenANewLeaseManagerIsUp() throws Exception {
-
- final String TEST_TSO_LEASE_PATH = "/test2_tsolease";
- final String TEST_CURRENT_TSO_PATH = "/test2_currenttso";
-
- // Launch the master instance...
- TSOChannelHandler tsoChannelHandler1 = mock(TSOChannelHandler.class);
- TSOStateManager stateManager1 = mock(TSOStateManager.class);
- when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
- leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
- tsoChannelHandler1,
- stateManager1,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
-
- leaseManager1.startService();
-
- // ...let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ...so it should be the current holder of the lease
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
- assertTrue(leaseManager1.stillInLeasePeriod());
-
- // Then launch another instance...
- TSOChannelHandler tsoChannelHandler2 = mock(TSOChannelHandler.class);
- TSOStateManager stateManager2 = mock(TSOStateManager.class);
- when(stateManager2.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_2, DUMMY_EPOCH_2));
- leaseManager2 = new PausableLeaseManager(LEASE_MGR_ID_2,
- tsoChannelHandler2,
- stateManager2,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
- leaseManager2.startService();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... and after the period, the first instance should be still the holder
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
- assertTrue(leaseManager1.stillInLeasePeriod());
- assertFalse(leaseManager2.stillInLeasePeriod());
- }
-
- @Test(timeOut = 60_000)
- public void testLeaseHolderChangesWhenActiveLeaseManagerIsPaused() throws Exception {
-
- final String TEST_TSO_LEASE_PATH = "/test3_tsolease";
- final String TEST_CURRENT_TSO_PATH = "/test3_currenttso";
-
- // Launch the master instance...
- TSOChannelHandler tsoChannelHandler1 = mock(TSOChannelHandler.class);
- TSOStateManager stateManager1 = mock(TSOStateManager.class);
- when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
- leaseManager1 = new PausableLeaseManager(LEASE_MGR_ID_1,
- tsoChannelHandler1,
- stateManager1,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
-
- leaseManager1.startService();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... so it should be the current holder of the lease
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "1");
- assertTrue(leaseManager1.stillInLeasePeriod());
-
- // Then launch another instance...
- TSOChannelHandler tsoChannelHandler2 = mock(TSOChannelHandler.class);
- TSOStateManager stateManager2 = mock(TSOStateManager.class);
- when(stateManager2.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_2, DUMMY_EPOCH_2));
- leaseManager2 = new PausableLeaseManager(LEASE_MGR_ID_2,
- tsoChannelHandler2,
- stateManager2,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
- leaseManager2.startService();
-
- // ... and pause active lease manager...
- leaseManager1.pausedInStillInLeasePeriod();
-
- // ... and let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... and check that lease owner should have changed to the second instance
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_2);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_2 + "2");
- assertTrue(leaseManager2.stillInLeasePeriod());
-
- // Now, lets resume the first instance...
- when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_3, DUMMY_EPOCH_3));
- leaseManager1.resume();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // and check the lease owner is still the second instance (preserves the lease)
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_2);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_2 + "2");
- assertFalse(leaseManager1.stillInLeasePeriod());
- assertTrue(leaseManager2.stillInLeasePeriod());
-
- // Finally, pause active lease manager when trying to renew lease...
- leaseManager2.pausedInTryToRenewLeasePeriod();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... and check lease owner is has changed again to the first instance
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "3");
- assertFalse(leaseManager2.stillInLeasePeriod());
- assertTrue(leaseManager1.stillInLeasePeriod());
-
- // Resume the second instance...
- leaseManager2.resume();
-
- // ... let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- // ... but the lease owner should still be the first instance
- checkLeaseHolder(TEST_TSO_LEASE_PATH, LEASE_MGR_ID_1);
- checkInstanceId(TEST_CURRENT_TSO_PATH, INSTANCE_ID_1 + "3");
- assertFalse(leaseManager2.stillInLeasePeriod());
- assertTrue(leaseManager1.stillInLeasePeriod());
-
- }
-
-
- @Test(timeOut = 40_000)
- public void testLeaseManagerPanicsWhenUnexpectedInfoIsFoundInCurrentTSOZnode() throws Exception {
-
- final String TEST_TSO_LEASE_PATH = "/test_wronginfo_tsolease";
- final String TEST_CURRENT_TSO_PATH = "/test_wronginfo_currenttso";
-
- Panicker panicker = spy(new MockPanicker());
-
- // Launch the master instance...
- TSOStateManager stateManager1 = mock(TSOStateManager.class);
- when(stateManager1.reset()).thenReturn(new TSOState(DUMMY_LOW_WATERMARK_1, DUMMY_EPOCH_1));
- PausableLeaseManager leaseManager = new PausableLeaseManager(LEASE_MGR_ID_1,
- mock(TSOChannelHandler.class),
- stateManager1,
- TEST_LEASE_PERIOD_IN_MS,
- TEST_TSO_LEASE_PATH,
- TEST_CURRENT_TSO_PATH,
- zkClient,
- panicker);
-
- leaseManager.startService();
- // ...and let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- leaseManager.pausedInTryToRenewLeasePeriod();
-
- // 1st Panic test) Inject corrupted data in the ZNode, force reelection and test the panicker is exercised
- zkClient.setData().forPath(TEST_CURRENT_TSO_PATH, "CorruptedData!!!".getBytes());
-
- // ...and let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
- leaseManager.resume();
- // ...and let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- ArgumentCaptor<IllegalArgumentException> trowableIAE = ArgumentCaptor.forClass(IllegalArgumentException.class);
- verify(panicker).panic(anyString(), trowableIAE.capture());
- assertTrue(trowableIAE.getValue() instanceof IllegalArgumentException);
- assertTrue(trowableIAE.getValue().getMessage().contains("Incorrect TSO Info found"));
-
- // 2nd Panic test) Simulate that a new master appeared in the meantime, force reelection
- // and test the panicker is exercised
- reset(panicker);
- zkClient.setData().forPath(TEST_CURRENT_TSO_PATH, "newTSO:12345#10000".getBytes());
-
- leaseManager.pausedInTryToRenewLeasePeriod();
-
- // ...and let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
- leaseManager.resume();
- // ...and let the test run for some time...
- Thread.sleep(TEST_LEASE_PERIOD_IN_MS * 2);
-
- ArgumentCaptor<LeaseManagement.LeaseManagementException> trowableLME =
- ArgumentCaptor.forClass(LeaseManagement.LeaseManagementException.class);
- verify(panicker).panic(anyString(), trowableLME.capture());
- assertTrue(trowableLME.getValue() instanceof LeaseManagement.LeaseManagementException);
- assertTrue(trowableLME.getValue().getMessage().contains("Another TSO replica was found"));
- }
-
- @Test(timeOut = 1000)
- public void testNonHALeaseManager() throws Exception {
-
- // Launch the instance...
- VoidLeaseManager leaseManager = new VoidLeaseManager(mock(TSOChannelHandler.class),
- mock(TSOStateManager.class));
-
- leaseManager.startService();
- assertTrue(leaseManager.stillInLeasePeriod());
- leaseManager.stopService();
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Checkers
- // ----------------------------------------------------------------------------------------------------------------
-
- private void checkLeaseHolder(String tsoLeasePath, String expectedLeaseHolder) throws Exception {
- byte[] leaseHolderInBytes = zkClient.getData().forPath(tsoLeasePath);
- String leaseHolder = new String(leaseHolderInBytes, Charsets.UTF_8);
-
- assertEquals(leaseHolder, expectedLeaseHolder);
- }
-
- private void checkInstanceId(String currentTSOPath, String expectedInstanceId) throws Exception {
- byte[] expectedInstanceIdInBytes = zkClient.getData().forPath(currentTSOPath);
- String instanceId = new String(expectedInstanceIdInBytes, Charsets.UTF_8);
-
- assertEquals(instanceId, expectedInstanceId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
deleted file mode 100644
index 6e54302..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestLongCache.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class TestLongCache {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestLongCache.class);
-
- private static final long TEST_VALUE = 1000;
-
- private Random random = new Random(System.currentTimeMillis());
-
- @Test
- public void testAddAndGetElemsAndResetCache() {
-
- // Cache configuration
- final int CACHE_SIZE = 10_000_000;
- final int CACHE_ASSOCIATIVITY = 32;
- Cache cache = new LongCache(CACHE_SIZE, CACHE_ASSOCIATIVITY);
-
- // After creation, cache values should be the default
- for (int i = 0; i < 1000; i++) {
- long position = random.nextLong();
- assertEquals(cache.get(position), LongCache.RESET_VALUE);
- }
-
- Set<Long> testedKeys = new TreeSet<>();
- // Populate some of the values
- for (int i = 0; i < 1000; i++) {
- long position = random.nextLong();
- cache.set(position, TEST_VALUE);
- testedKeys.add(position);
- }
-
- // Get the values and check them
- for (long key : testedKeys) {
- assertEquals(cache.get(key), TEST_VALUE);
- }
-
- // Reset cache and check the values are the default again
- long startTimeInMs = System.currentTimeMillis();
- cache.reset();
- long endTimeInMs = System.currentTimeMillis();
- long resetTimeInMs = endTimeInMs - startTimeInMs;
- LOG.info("Time in reseting cache of {}/{} elems/asoc {}ms", CACHE_SIZE, CACHE_ASSOCIATIVITY, resetTimeInMs);
-
- for (long key : testedKeys) {
- assertEquals(cache.get(key), LongCache.RESET_VALUE);
- }
-
- }
-
- @Test(timeOut = 10000)
- public void testEntriesAge() {
-
- final int entries = 1000;
-
- Cache cache = new LongCache(entries, 16);
-
- int removals = 0;
- long totalAge = 0;
- double tempStdDev = 0;
- double tempAvg = 0;
-
- int i = 0;
- int largestDeletedTimestamp = 0;
- for (; i < entries * 10; ++i) {
- long removed = cache.set(random.nextLong(), i);
- if (removed > largestDeletedTimestamp) {
- largestDeletedTimestamp = (int) removed;
- }
- }
-
- long time = System.nanoTime();
- for (; i < entries * 100; ++i) {
- long removed = cache.set(random.nextLong(), i);
- if (removed > largestDeletedTimestamp) {
- largestDeletedTimestamp = (int) removed;
- }
- int gap = i - ((int) largestDeletedTimestamp);
- removals++;
- totalAge += gap;
- double oldAvg = tempAvg;
- tempAvg += (gap - tempAvg) / removals;
- tempStdDev += (gap - oldAvg) * (gap - tempAvg);
- }
- long elapsed = System.nanoTime() - time;
- LOG.info("Elapsed (ms): " + (elapsed / (double) 1000));
-
- double avgGap = totalAge / (double) removals;
- LOG.info("Avg gap: " + (tempAvg));
- LOG.info("Std dev gap: " + Math.sqrt((tempStdDev / entries)));
- assertTrue("avgGap should be greater than entries * 0.6",
- avgGap > entries * 0.6);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
deleted file mode 100644
index c9bd743..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestPanicker.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-
-@SuppressWarnings({"UnusedDeclaration"})
-public class TestPanicker {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestPanicker.class);
-
- @Mock
- private CommitTable.Writer mockWriter;
- @Mock
- private MetricsRegistry metrics;
-
- @BeforeMethod
- public void initMocksAndComponents() {
- MockitoAnnotations.initMocks(this);
- }
-
- @AfterMethod
- void afterMethod() {
- Mockito.reset(mockWriter);
- }
-
- // Note this test has been moved and refactored to TestTimestampOracle because
- // it tests the behaviour of the TimestampOracle.
- // Please, remove me in a future commit
- @Test
- public void testTimestampOraclePanic() throws Exception {
- TimestampStorage storage = spy(new TimestampOracleImpl.InMemoryTimestampStorage());
- Panicker panicker = spy(new MockPanicker());
-
- doThrow(new RuntimeException("Out of memory")).when(storage).updateMaxTimestamp(anyLong(), anyLong());
-
- final TimestampOracleImpl tso = new TimestampOracleImpl(metrics, storage, panicker);
- tso.initialize();
- Thread allocThread = new Thread("AllocThread") {
- @Override
- public void run() {
- try {
- while (true) {
- tso.next();
- }
- } catch (IOException ioe) {
- LOG.error("Shouldn't occur");
- }
- }
- };
- allocThread.start();
-
- verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
- }
-
- // Note this test has been moved and refactored to TestPersistenceProcessor because
- // it tests the behaviour of the PersistenceProcessor.
- // Please, remove me in a future commit
- @Test
- public void testCommitTablePanic() throws Exception {
- Panicker panicker = spy(new MockPanicker());
-
- doThrow(new IOException("Unable to write@TestPanicker")).when(mockWriter).flush();
-
- final CommitTable.Client mockClient = mock(CommitTable.Client.class);
- CommitTable commitTable = new CommitTable() {
- @Override
- public Writer getWriter() {
- return mockWriter;
- }
-
- @Override
- public Client getClient() {
- return mockClient;
- }
- };
-
- LeaseManager leaseManager = mock(LeaseManager.class);
- doReturn(true).when(leaseManager).stillInLeasePeriod();
- PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
- metrics,
- "localhost:1234",
- leaseManager,
- commitTable,
- mock(ReplyProcessor.class),
- mock(RetryProcessor.class),
- panicker);
- proc.persistCommit(1, 2, null, new MonitoringContext(metrics));
- verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
- }
-
- // Note this test has been moved and refactored to TestPersistenceProcessor because
- // it tests the behaviour of the PersistenceProcessor.
- // Please, remove me in a future commit
- @Test
- public void testRuntimeExceptionTakesDownDaemon() throws Exception {
- Panicker panicker = spy(new MockPanicker());
-
- final CommitTable.Writer mockWriter = mock(CommitTable.Writer.class);
- doThrow(new RuntimeException("Kaboom!")).when(mockWriter).addCommittedTransaction(anyLong(), anyLong());
-
- final CommitTable.Client mockClient = mock(CommitTable.Client.class);
- CommitTable commitTable = new CommitTable() {
- @Override
- public Writer getWriter() {
- return mockWriter;
- }
-
- @Override
- public Client getClient() {
- return mockClient;
- }
- };
- PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
- metrics,
- "localhost:1234",
- mock(LeaseManager.class),
- commitTable,
- mock(ReplyProcessor.class),
- mock(RetryProcessor.class),
- panicker);
- proc.persistCommit(1, 2, null, new MonitoringContext(metrics));
- verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
deleted file mode 100644
index 03d4ffc..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestPersistenceProcessor.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.tso.PersistenceProcessorImpl.Batch;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.mockito.MockitoAnnotations;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-
-@SuppressWarnings({"UnusedDeclaration"})
-public class TestPersistenceProcessor {
-
- @Mock
- private Batch batch;
- @Mock
- private CommitTable.Writer mockWriter;
- @Mock
- private CommitTable.Client mockClient;
- @Mock
- private RetryProcessor retryProcessor;
- @Mock
- private ReplyProcessor replyProcessor;
- @Mock
- private Panicker panicker;
-
- private MetricsRegistry metrics;
- private CommitTable commitTable;
-
- @BeforeMethod(alwaysRun = true, timeOut = 30_000)
- public void initMocksAndComponents() throws Exception {
-
- MockitoAnnotations.initMocks(this);
-
- // Configure mock writer to flush successfully
- doThrow(new IOException("Unable to write")).when(mockWriter).flush();
-
- // Configure null metrics provider
- metrics = new NullMetricsProvider();
-
- // Configure commit table to return the mocked writer and client
- commitTable = new CommitTable() {
- @Override
- public Writer getWriter() {
- return mockWriter;
- }
-
- @Override
- public Client getClient() {
- return mockClient;
- }
- };
- }
-
- @AfterMethod
- void afterMethod() {
- Mockito.reset(mockWriter);
- }
-
- @Test
- public void testCommitPersistenceWithNonHALeaseManager() throws Exception {
-
- // Init a non-HA lease manager
- VoidLeaseManager leaseManager = spy(new VoidLeaseManager(mock(TSOChannelHandler.class),
- mock(TSOStateManager.class)));
-
- TSOServerConfig tsoServerConfig = new TSOServerConfig();
- tsoServerConfig.setBatchPersistTimeoutInMs(100);
- // Component under test
- PersistenceProcessor proc = new PersistenceProcessorImpl(tsoServerConfig,
- metrics,
- "localhost:1234",
- batch,
- leaseManager,
- commitTable,
- replyProcessor,
- retryProcessor,
- panicker);
-
- // The non-ha lease manager always return true for
- // stillInLeasePeriod(), so verify the batch sends replies as master
- MonitoringContext monCtx = new MonitoringContext(metrics);
- proc.persistCommit(1, 2, null, monCtx);
- verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
- verify(batch, timeout(1000).times(2)).sendRepliesAndReset(any(ReplyProcessor.class),
- any(RetryProcessor.class),
- eq(true));
- }
-
- @Test
- public void testCommitPersistenceWithHALeaseManager() throws Exception {
-
- // Init a HA lease manager
- LeaseManager leaseManager = mock(LeaseManager.class);
-
- TSOServerConfig tsoServerConfig = new TSOServerConfig();
- tsoServerConfig.setBatchPersistTimeoutInMs(100);
- // Component under test
- PersistenceProcessor proc = new PersistenceProcessorImpl(tsoServerConfig,
- metrics,
- "localhost:1234",
- batch,
- leaseManager,
- commitTable,
- replyProcessor,
- retryProcessor,
- panicker);
-
- // Configure the lease manager to always return true for
- // stillInLeasePeriod, so verify the batch sends replies as master
- doReturn(true).when(leaseManager).stillInLeasePeriod();
- MonitoringContext monCtx = new MonitoringContext(metrics);
- proc.persistCommit(1, 2, null, monCtx);
- verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
- verify(batch).sendRepliesAndReset(any(ReplyProcessor.class), any(RetryProcessor.class), eq(true));
-
- // Configure the lease manager to always return true first and false
- // later for stillInLeasePeriod, so verify the batch sends replies as
- // non-master
- reset(leaseManager);
- reset(batch);
- doReturn(true).doReturn(false).when(leaseManager).stillInLeasePeriod();
- proc.persistCommit(1, 2, null, monCtx);
- verify(leaseManager, timeout(1000).times(2)).stillInLeasePeriod();
- verify(batch).sendRepliesAndReset(any(ReplyProcessor.class), any(RetryProcessor.class), eq(false));
-
- // Configure the lease manager to always return false for
- // stillInLeasePeriod, so verify the batch sends replies as non-master
- reset(leaseManager);
- reset(batch);
- doReturn(false).when(leaseManager).stillInLeasePeriod();
- proc.persistCommit(1, 2, null, monCtx);
- verify(leaseManager, timeout(1000).times(1)).stillInLeasePeriod();
- verify(batch).sendRepliesAndReset(any(ReplyProcessor.class), any(RetryProcessor.class), eq(false));
-
- }
-
- @Test
- public void testCommitTableExceptionOnCommitPersistenceTakesDownDaemon() throws Exception {
-
- // Init lease management (doesn't matter if HA or not)
- LeaseManagement leaseManager = mock(LeaseManagement.class);
- PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
- metrics,
- "localhost:1234",
- leaseManager,
- commitTable,
- mock(ReplyProcessor.class),
- mock(RetryProcessor.class),
- panicker);
- MonitoringContext monCtx = new MonitoringContext(metrics);
-
- // Configure lease manager to work normally
- doReturn(true).when(leaseManager).stillInLeasePeriod();
-
- // Configure commit table writer to explode when flushing changes to DB
- doThrow(new IOException("Unable to write@TestPersistenceProcessor2")).when(mockWriter).flush();
-
- // Check the panic is extended!
- proc.persistCommit(1, 2, null, monCtx);
- verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
-
- }
-
- @Test
- public void testRuntimeExceptionOnCommitPersistenceTakesDownDaemon() throws Exception {
-
- PersistenceProcessor proc = new PersistenceProcessorImpl(new TSOServerConfig(),
- metrics,
- "localhost:1234",
- mock(LeaseManagement.class),
- commitTable,
- replyProcessor,
- retryProcessor,
- panicker);
-
- // Configure writer to explode with a runtime exception
- doThrow(new RuntimeException("Kaboom!")).when(mockWriter).addCommittedTransaction(anyLong(), anyLong());
- MonitoringContext monCtx = new MonitoringContext(metrics);
-
- // Check the panic is extended!
- proc.persistCommit(1, 2, null, monCtx);
- verify(panicker, timeout(1000).atLeastOnce()).panic(anyString(), any(Throwable.class));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
deleted file mode 100644
index d3e2c18..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestRequestProcessor.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.collect.Lists;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import org.jboss.netty.channel.Channel;
-import org.mockito.ArgumentCaptor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.Collections;
-import java.util.List;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class TestRequestProcessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestRequestProcessor.class);
-
- private static final int CONFLICT_MAP_SIZE = 1000;
- private static final int CONFLICT_MAP_ASSOCIATIVITY = 32;
-
- private MetricsRegistry metrics = new NullMetricsProvider();
-
- private PersistenceProcessor persist;
-
- private TSOStateManager stateManager;
-
- // Request processor under test
- private RequestProcessor requestProc;
-
- @BeforeMethod
- public void beforeMethod() throws Exception {
-
- // Build the required scaffolding for the test
- MetricsRegistry metrics = new NullMetricsProvider();
-
- TimestampOracleImpl timestampOracle =
- new TimestampOracleImpl(metrics, new TimestampOracleImpl.InMemoryTimestampStorage(), new MockPanicker());
-
- stateManager = new TSOStateManagerImpl(timestampOracle);
-
- persist = mock(PersistenceProcessor.class);
-
- TSOServerConfig config = new TSOServerConfig();
- config.setMaxItems(CONFLICT_MAP_SIZE);
-
- requestProc = new RequestProcessorImpl(config, metrics, timestampOracle, persist, new MockPanicker());
-
- // Initialize the state for the experiment
- stateManager.register(requestProc);
- stateManager.reset();
-
- }
-
- @Test(timeOut = 30_000)
- public void testTimestamp() throws Exception {
-
- requestProc.timestampRequest(null, new MonitoringContext(metrics));
- ArgumentCaptor<Long> firstTScapture = ArgumentCaptor.forClass(Long.class);
- verify(persist, timeout(100).times(1)).persistTimestamp(
- firstTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
-
- long firstTS = firstTScapture.getValue();
- // verify that timestamps increase monotonically
- for (int i = 0; i < 100; i++) {
- requestProc.timestampRequest(null, new MonitoringContext(metrics));
- verify(persist, timeout(100).times(1)).persistTimestamp(eq(firstTS++), any(Channel.class), any(MonitoringContext.class));
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testCommit() throws Exception {
-
- requestProc.timestampRequest(null, new MonitoringContext(metrics));
- ArgumentCaptor<Long> TScapture = ArgumentCaptor.forClass(Long.class);
- verify(persist, timeout(100).times(1)).persistTimestamp(
- TScapture.capture(), any(Channel.class), any(MonitoringContext.class));
- long firstTS = TScapture.getValue();
-
- List<Long> writeSet = Lists.newArrayList(1L, 20L, 203L);
- requestProc.commitRequest(firstTS - 1, writeSet, false, null, new MonitoringContext(metrics));
- verify(persist, timeout(100).times(1)).persistAbort(eq(firstTS - 1), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
-
- requestProc.commitRequest(firstTS, writeSet, false, null, new MonitoringContext(metrics));
- ArgumentCaptor<Long> commitTScapture = ArgumentCaptor.forClass(Long.class);
-
- verify(persist, timeout(100).times(1)).persistCommit(eq(firstTS), commitTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
- assertTrue("Commit TS must be greater than start TS", commitTScapture.getValue() > firstTS);
-
- // test conflict
- requestProc.timestampRequest(null, new MonitoringContext(metrics));
- TScapture = ArgumentCaptor.forClass(Long.class);
- verify(persist, timeout(100).times(2)).persistTimestamp(
- TScapture.capture(), any(Channel.class), any(MonitoringContext.class));
- long secondTS = TScapture.getValue();
-
- requestProc.timestampRequest(null, new MonitoringContext(metrics));
- TScapture = ArgumentCaptor.forClass(Long.class);
- verify(persist, timeout(100).times(3)).persistTimestamp(
- TScapture.capture(), any(Channel.class), any(MonitoringContext.class));
- long thirdTS = TScapture.getValue();
-
- requestProc.commitRequest(thirdTS, writeSet, false, null, new MonitoringContext(metrics));
- verify(persist, timeout(100).times(1)).persistCommit(eq(thirdTS), anyLong(), any(Channel.class), any(MonitoringContext.class));
- requestProc.commitRequest(secondTS, writeSet, false, null, new MonitoringContext(metrics));
- verify(persist, timeout(100).times(1)).persistAbort(eq(secondTS), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
-
- }
-
- @Test(timeOut = 30_000)
- public void testCommitRequestAbortsWhenResettingRequestProcessorState() throws Exception {
-
- List<Long> writeSet = Collections.emptyList();
-
- // Start a transaction...
- requestProc.timestampRequest(null, new MonitoringContext(metrics));
- ArgumentCaptor<Long> capturedTS = ArgumentCaptor.forClass(Long.class);
- verify(persist, timeout(100).times(1)).persistTimestamp(capturedTS.capture(),
- any(Channel.class),
- any(MonitoringContext.class));
- long startTS = capturedTS.getValue();
-
- // ... simulate the reset of the RequestProcessor state (e.g. due to
- // a change in mastership) and...
- stateManager.reset();
-
- // ...check that the transaction is aborted when trying to commit
- requestProc.commitRequest(startTS, writeSet, false, null, new MonitoringContext(metrics));
- verify(persist, timeout(100).times(1)).persistAbort(eq(startTS), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
-
- }
-
- @Test(timeOut = 5_000)
- public void testLowWatermarkIsStoredOnlyWhenACacheElementIsEvicted() throws Exception {
-
- final int ANY_START_TS = 1;
- final long FIRST_COMMIT_TS_EVICTED = 1L;
- final long NEXT_COMMIT_TS_THAT_SHOULD_BE_EVICTED = 2L;
-
- // Fill the cache to provoke a cache eviction
- for (long i = 0; i < CONFLICT_MAP_SIZE + CONFLICT_MAP_ASSOCIATIVITY; i++) {
- long writeSetElementHash = i + 1; // This is to match the assigned CT: K/V in cache = WS Element Hash/CT
- List<Long> writeSet = Lists.newArrayList(writeSetElementHash);
- requestProc.commitRequest(ANY_START_TS, writeSet, false, null, new MonitoringContext(metrics));
- }
-
- Thread.currentThread().sleep(3000); // Allow the Request processor to finish the request processing
-
- // Check that first time its called is on init
- verify(persist, timeout(100).times(1)).persistLowWatermark(0L);
- // Then, check it is called when cache is full and the first element is evicted (should be a 1)
- verify(persist, timeout(100).times(1)).persistLowWatermark(FIRST_COMMIT_TS_EVICTED);
- // Finally it should never be called with the next element
- verify(persist, timeout(100).never()).persistLowWatermark(NEXT_COMMIT_TS_THAT_SHOULD_BE_EVICTED);
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
deleted file mode 100644
index decd75e..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestRetryProcessor.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.base.Optional;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.committable.InMemoryCommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import org.jboss.netty.channel.Channel;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-import static org.testng.AssertJUnit.assertEquals;
-
-public class TestRetryProcessor {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestRetryProcessor.class);
-
- private MetricsRegistry metrics = new NullMetricsProvider();
-
- private static long NON_EXISTING_ST_TX = 1000;
- private static long ST_TX_1 = 0;
- private static long CT_TX_1 = 1;
- private static long ST_TX_2 = 2;
-
- @Mock
- private Channel channel;
- @Mock
- private ReplyProcessor replyProc;
- @Mock
- private Panicker panicker;
-
- private CommitTable commitTable;
-
- @BeforeMethod(alwaysRun = true, timeOut = 30_000)
- public void initMocksAndComponents() {
- MockitoAnnotations.initMocks(this);
- // Init components
- commitTable = new InMemoryCommitTable();
- metrics = new NullMetricsProvider();
-
- }
-
- @Test(timeOut = 10_000)
- public void testBasicFunctionality() throws Exception {
-
- // The element to test
- RetryProcessor retryProc = new RetryProcessorImpl(metrics, commitTable, replyProc, panicker);
-
- // Test we'll reply with an abort for a retry request when the start timestamp IS NOT in the commit table
- retryProc.disambiguateRetryRequestHeuristically(NON_EXISTING_ST_TX, channel, new MonitoringContext(metrics));
- ArgumentCaptor<Long> firstTScapture = ArgumentCaptor.forClass(Long.class);
- verify(replyProc, timeout(100).times(1)).abortResponse(firstTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
-
- long startTS = firstTScapture.getValue();
- assertEquals("Captured timestamp should be the same as NON_EXISTING_ST_TX", NON_EXISTING_ST_TX, startTS);
-
- // Test we'll reply with a commit for a retry request when the start timestamp IS in the commit table
- commitTable.getWriter().addCommittedTransaction(ST_TX_1, CT_TX_1); // Add a tx to commit table
-
- retryProc.disambiguateRetryRequestHeuristically(ST_TX_1, channel, new MonitoringContext(metrics));
- ArgumentCaptor<Long> secondTScapture = ArgumentCaptor.forClass(Long.class);
- verify(replyProc, timeout(100).times(1))
- .commitResponse(eq(false), firstTScapture.capture(), secondTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
-
- startTS = firstTScapture.getValue();
- long commitTS = secondTScapture.getValue();
- assertEquals("Captured timestamp should be the same as ST_TX_1", ST_TX_1, startTS);
- assertEquals("Captured timestamp should be the same as CT_TX_1", CT_TX_1, commitTS);
- }
-
- @Test(timeOut = 10_000)
- public void testRetriedRequestForInvalidatedTransactionReturnsAnAbort() throws Exception {
-
- // Invalidate the transaction
- commitTable.getClient().tryInvalidateTransaction(ST_TX_1);
-
- // Pre-start verification: Validate that the transaction is invalidated
- // NOTE: This test should be in the a test class for InMemoryCommitTable
- Optional<CommitTimestamp> invalidTxMarker = commitTable.getClient().getCommitTimestamp(ST_TX_1).get();
- Assert.assertTrue(invalidTxMarker.isPresent());
- Assert.assertEquals(invalidTxMarker.get().getValue(), InMemoryCommitTable.INVALID_TRANSACTION_MARKER);
-
- // The element to test
- RetryProcessor retryProc = new RetryProcessorImpl(metrics, commitTable, replyProc, panicker);
-
- // Test we'll reply with an abort for a retry request when the
- // transaction id IS in the commit table BUT invalidated
- retryProc.disambiguateRetryRequestHeuristically(ST_TX_1, channel, new MonitoringContext(metrics));
- ArgumentCaptor<Long> startTScapture = ArgumentCaptor.forClass(Long.class);
- verify(replyProc, timeout(100).times(1)).abortResponse(startTScapture.capture(), any(Channel.class), any(MonitoringContext.class));
-
- long startTS = startTScapture.getValue();
- Assert.assertEquals(startTS, ST_TX_1, "Captured timestamp should be the same as NON_EXISTING_ST_TX");
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
deleted file mode 100644
index 3f071c0..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOChannelHandlerNetty.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.proto.TSOProto;
-import org.jboss.netty.bootstrap.ClientBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelException;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
-import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
-import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
-import static org.mockito.Matchers.anyCollectionOf;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-
-@SuppressWarnings({"UnusedDeclaration", "StatementWithEmptyBody"})
-public class TestTSOChannelHandlerNetty {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTSOChannelHandlerNetty.class);
-
- @Mock
- private
- RequestProcessor requestProcessor;
-
- // Component under test
- private TSOChannelHandler channelHandler;
-
- @BeforeMethod
- public void beforeTestMethod() {
- MockitoAnnotations.initMocks(this);
- TSOServerConfig config = new TSOServerConfig();
- config.setPort(1434);
- channelHandler = new TSOChannelHandler(config, requestProcessor, new NullMetricsProvider());
- }
-
- @AfterMethod
- public void afterTestMethod() throws IOException {
- channelHandler.close();
- }
-
- @Test(timeOut = 10_000)
- public void testMainAPI() throws Exception {
-
- // Check initial state
- assertNull(channelHandler.listeningChannel);
- assertNull(channelHandler.channelGroup);
-
- // Check initial connection
- channelHandler.reconnect();
- assertTrue(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 1);
- assertEquals(((InetSocketAddress) channelHandler.listeningChannel.getLocalAddress()).getPort(), 1434);
-
- // Check connection close
- channelHandler.closeConnection();
- assertFalse(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 0);
-
- // Check re-closing connection
- channelHandler.closeConnection();
- assertFalse(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 0);
-
- // Check connection after closing
- channelHandler.reconnect();
- assertTrue(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 1);
-
- // Check re-connection
- channelHandler.reconnect();
- assertTrue(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 1);
-
- // Exercise closeable with re-connection trial
- channelHandler.close();
- assertFalse(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 0);
- try {
- channelHandler.reconnect();
- } catch (ChannelException e) {
- // Expected: Can't reconnect after closing
- assertFalse(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 0);
- }
-
- }
-
- @Test(timeOut = 10_000)
- public void testNettyConnectionToTSOFromClient() throws Exception {
-
- ClientBootstrap nettyClient = createNettyClientBootstrap();
-
- ChannelFuture channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
-
- // ------------------------------------------------------------------------------------------------------------
- // Test the client can't connect cause the server is not there
- // ------------------------------------------------------------------------------------------------------------
- while (!channelF.isDone()) /** do nothing */ ;
- assertFalse(channelF.isSuccess());
-
- // ------------------------------------------------------------------------------------------------------------
- // Test creation of a server connection
- // ------------------------------------------------------------------------------------------------------------
- channelHandler.reconnect();
- assertTrue(channelHandler.listeningChannel.isOpen());
- // Eventually the channel group of the server should contain the listening channel
- assertEquals(channelHandler.channelGroup.size(), 1);
-
- // ------------------------------------------------------------------------------------------------------------
- // Test that a client can connect now
- // ------------------------------------------------------------------------------------------------------------
- channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
- while (!channelF.isDone()) /** do nothing */ ;
- assertTrue(channelF.isSuccess());
- assertTrue(channelF.getChannel().isConnected());
- // Eventually the channel group of the server should have 2 elements
- while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
-
- // ------------------------------------------------------------------------------------------------------------
- // Close the channel on the client side and test we have one element less in the channel group
- // ------------------------------------------------------------------------------------------------------------
- channelF.getChannel().close().await();
- // Eventually the channel group of the server should have only one element
- while (channelHandler.channelGroup.size() != 1) /** do nothing */ ;
-
- // ------------------------------------------------------------------------------------------------------------
- // Open a new channel and test the connection closing on the server side through the channel handler
- // ------------------------------------------------------------------------------------------------------------
- channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
- while (!channelF.isDone()) /** do nothing */ ;
- assertTrue(channelF.isSuccess());
- // Eventually the channel group of the server should have 2 elements again
- while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
- channelHandler.closeConnection();
- assertFalse(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 0);
- // Wait some time and check the channel was closed
- TimeUnit.SECONDS.sleep(1);
- assertFalse(channelF.getChannel().isOpen());
-
- // ------------------------------------------------------------------------------------------------------------
- // Test server re-connections with connected clients
- // ------------------------------------------------------------------------------------------------------------
- // Connect first time
- channelHandler.reconnect();
- assertTrue(channelHandler.listeningChannel.isOpen());
- // Eventually the channel group of the server should contain the listening channel
- assertEquals(channelHandler.channelGroup.size(), 1);
- // Check the client can connect
- channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
- while (!channelF.isDone()) /** do nothing */ ;
- assertTrue(channelF.isSuccess());
- // Eventually the channel group of the server should have 2 elements
- while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
- // Re-connect and check that client connection was gone
- channelHandler.reconnect();
- assertTrue(channelHandler.listeningChannel.isOpen());
- // Eventually the channel group of the server should contain the listening channel
- assertEquals(channelHandler.channelGroup.size(), 1);
- // Wait some time and check the channel was closed
- TimeUnit.SECONDS.sleep(1);
- assertFalse(channelF.getChannel().isOpen());
-
- // ------------------------------------------------------------------------------------------------------------
- // Test closeable interface with re-connection trial
- // ------------------------------------------------------------------------------------------------------------
- channelHandler.close();
- assertFalse(channelHandler.listeningChannel.isOpen());
- assertEquals(channelHandler.channelGroup.size(), 0);
- }
-
- @Test(timeOut = 10_000)
- public void testNettyChannelWriting() throws Exception {
-
- // ------------------------------------------------------------------------------------------------------------
- // Prepare test
- // ------------------------------------------------------------------------------------------------------------
-
- // Connect channel handler
- channelHandler.reconnect();
- // Create client and connect it
- ClientBootstrap nettyClient = createNettyClientBootstrap();
- ChannelFuture channelF = nettyClient.connect(new InetSocketAddress("localhost", 1434));
- // Basic checks for connection
- while (!channelF.isDone()) /** do nothing */ ;
- assertTrue(channelF.isSuccess());
- assertTrue(channelF.getChannel().isConnected());
- Channel channel = channelF.getChannel();
- // Eventually the channel group of the server should have 2 elements
- while (channelHandler.channelGroup.size() != 2) /** do nothing */ ;
- // Write first handshake request
- TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
- // NOTE: Add here the required handshake capabilities when necessary
- handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
- channelF.getChannel().write(TSOProto.Request.newBuilder().setHandshakeRequest(handshake.build()).build());
-
- // ------------------------------------------------------------------------------------------------------------
- // Test channel writing
- // ------------------------------------------------------------------------------------------------------------
- testWritingTimestampRequest(channel);
-
- testWritingCommitRequest(channel);
- }
-
- private void testWritingTimestampRequest(Channel channel) throws InterruptedException {
- // Reset mock
- reset(requestProcessor);
- TSOProto.Request.Builder tsBuilder = TSOProto.Request.newBuilder();
- TSOProto.TimestampRequest.Builder tsRequestBuilder = TSOProto.TimestampRequest.newBuilder();
- tsBuilder.setTimestampRequest(tsRequestBuilder.build());
- // Write into the channel
- channel.write(tsBuilder.build()).await();
- verify(requestProcessor, timeout(100).times(1)).timestampRequest(any(Channel.class), any(MonitoringContext.class));
- verify(requestProcessor, timeout(100).never())
- .commitRequest(anyLong(), anyCollectionOf(Long.class), anyBoolean(), any(Channel.class), any(MonitoringContext.class));
- }
-
- private void testWritingCommitRequest(Channel channel) throws InterruptedException {
- // Reset mock
- reset(requestProcessor);
- TSOProto.Request.Builder commitBuilder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitRequestBuilder = TSOProto.CommitRequest.newBuilder();
- commitRequestBuilder.setStartTimestamp(666);
- commitRequestBuilder.addCellId(666);
- commitBuilder.setCommitRequest(commitRequestBuilder.build());
- TSOProto.Request r = commitBuilder.build();
- assertTrue(r.hasCommitRequest());
- // Write into the channel
- channel.write(commitBuilder.build()).await();
- verify(requestProcessor, timeout(100).never()).timestampRequest(any(Channel.class), any(MonitoringContext.class));
- verify(requestProcessor, timeout(100).times(1))
- .commitRequest(eq(666L), anyCollectionOf(Long.class), eq(false), any(Channel.class), any(MonitoringContext.class));
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private ClientBootstrap createNettyClientBootstrap() {
-
- ChannelFactory factory = new NioClientSocketChannelFactory(
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("client-boss-%d").build()),
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("client-worker-%d").build()), 1);
- // Create the bootstrap
- ClientBootstrap bootstrap = new ClientBootstrap(factory);
- bootstrap.setOption("tcpNoDelay", true);
- bootstrap.setOption("keepAlive", true);
- bootstrap.setOption("reuseAddress", true);
- bootstrap.setOption("connectTimeoutMillis", 100);
- ChannelPipeline pipeline = bootstrap.getPipeline();
- pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
- pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
- pipeline.addLast("protobufencoder", new ProtobufEncoder());
- pipeline.addLast("handler", new SimpleChannelHandler() {
-
- @Override
- public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
- LOG.info("Channel {} connected", ctx.getChannel());
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
- LOG.error("Error on channel {}", ctx.getChannel(), e.getCause());
- }
-
- });
- return bootstrap;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
deleted file mode 100644
index eda0985..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestTSOStateManager.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.yahoo.omid.tso.TSOStateManager.StateObserver;
-import com.yahoo.omid.tso.TSOStateManager.TSOState;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class TestTSOStateManager {
-
- private static final long INITIAL_STATE_VALUE = 1L;
- private static final long NEW_STATE_VALUE = 1000;
-
- // Mocks
- private TimestampOracle timestampOracle = mock(TimestampOracle.class);
-
- // Component under test
- private TSOStateManager stateManager = new TSOStateManagerImpl(timestampOracle);
-
- @BeforeMethod
- public void beforeMethod() {
- // Initialize the state with the one reported by the Timestamp Oracle
- when(timestampOracle.getLast()).thenReturn(INITIAL_STATE_VALUE);
- }
-
- @Test
- public void testResetOfTSOServerState() throws Exception {
-
- // Reset the state and check we get the initial state values
- TSOState initialState = stateManager.reset();
- assertEquals(initialState.getLowWatermark(), INITIAL_STATE_VALUE);
- assertEquals(initialState.getEpoch(), INITIAL_STATE_VALUE);
- assertTrue("In this implementation low watermark should be equal to epoch",
- initialState.getLowWatermark() == initialState.getEpoch());
-
- // Then, simulate a change in the state returned by the Timestamp Oracle...
- when(timestampOracle.getLast()).thenReturn(NEW_STATE_VALUE);
- // ... and again, reset the state and check we get the new values
- TSOState secondState = stateManager.reset();
- assertEquals(secondState.getLowWatermark(), NEW_STATE_VALUE);
- assertEquals(secondState.getEpoch(), NEW_STATE_VALUE);
- assertTrue("In this implementation low watermark should be equal to epoch",
- secondState.getLowWatermark() == secondState.getEpoch());
-
- }
-
- @Test
- public void testObserverRegistrationAndDeregistrationForStateChanges() throws Exception {
-
- // Register observer 1 for receiving state changes
- StateObserver observer1 = spy(new DummyObserver());
- stateManager.register(observer1);
-
- // Reset the state to trigger observer notifications
- TSOState state = stateManager.reset();
-
- // Check observer 1 was notified with the corresponding state
- verify(observer1, timeout(100).times(1)).update(eq(state));
-
- // Register observer 1 for receiving state changes
- StateObserver observer2 = spy(new DummyObserver());
- stateManager.register(observer2);
-
- // Again, reset the state to trigger observer notifications
- state = stateManager.reset();
-
- // Check both observers were notified with the corresponding state
- verify(observer1, timeout(100).times(1)).update(eq(state));
- verify(observer2, timeout(100).times(1)).update(eq(state));
-
- // De-register observer 1
- stateManager.unregister(observer1);
-
- // Again, reset the state to trigger observer notifications
- state = stateManager.reset();
-
- // Check only observer 2 was notified
- verify(observer1, timeout(100).times(0)).update(eq(state));
- verify(observer2, timeout(100).times(1)).update(eq(state));
- }
-
- // ------------------------------------------------------------------------
- // -------------------------- Helper classes ------------------------------
- // ------------------------------------------------------------------------
-
- private class DummyObserver implements StateObserver {
-
- @Override
- public void update(TSOState state) throws IOException {
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
deleted file mode 100644
index 3a6f1b9..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestTimestampOracle.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso;
-
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.atLeastOnce;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-public class TestTimestampOracle {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTimestampOracle.class);
-
- @Mock
- private MetricsRegistry metrics;
- @Mock
- private Panicker panicker;
- @Mock
- private TimestampStorage timestampStorage;
-
- // Component under test
- @InjectMocks
- private TimestampOracleImpl timestampOracle;
-
- @BeforeMethod(alwaysRun = true, timeOut = 30_000)
- public void initMocksAndComponents() {
- MockitoAnnotations.initMocks(this);
- }
-
- @Test(timeOut = 10_000)
- public void testMonotonicTimestampGrowth() throws Exception {
-
- // Intialize component under test
- timestampOracle.initialize();
-
- long last = timestampOracle.next();
- for (int i = 0; i < (3 * TimestampOracleImpl.TIMESTAMP_BATCH); i++) {
- long current = timestampOracle.next();
- assertEquals(current, last + 1, "Not monotonic growth");
- last = current;
- }
- assertTrue(timestampOracle.getLast() == last);
- LOG.info("Last timestamp: {}", last);
- }
-
- @Test(timeOut = 10_000)
- public void testTimestampOraclePanicsWhenTheStorageHasProblems() throws Exception {
-
- // Intialize component under test
- timestampOracle.initialize();
-
- // Cause an exception when updating the max timestamp
- final CountDownLatch updateMaxTimestampMethodCalled = new CountDownLatch(1);
- doAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
- updateMaxTimestampMethodCalled.countDown();
- throw new RuntimeException("Out of memory or something");
- }
- }).when(timestampStorage).updateMaxTimestamp(anyLong(), anyLong());
-
- // Make the previous exception to be thrown
- Thread allocThread = new Thread("AllocThread") {
- @Override
- public void run() {
- try {
- while (true) {
- timestampOracle.next();
- }
- } catch (IOException ioe) {
- LOG.error("Shouldn't occur");
- }
- }
- };
- allocThread.start();
-
- updateMaxTimestampMethodCalled.await();
-
- // Verify that it has blown up
- verify(panicker, atLeastOnce()).panic(anyString(), any(Throwable.class));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java
deleted file mode 100644
index eeff214..0000000
--- a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-import com.yahoo.statemachine.StateMachine.FsmImpl;
-
-public class TSOClientAccessor {
-
- public static void closeChannel(TSOClient tsoClient) throws InterruptedException {
- FsmImpl fsm = (FsmImpl) tsoClient.fsm;
- TSOClient.ConnectedState connectedState = (TSOClient.ConnectedState) fsm.getState();
- connectedState.channel.close().await();
- }
-}
[11/52] [abbrv] incubator-omid git commit: Rename tsoclient package
to tso.client
Posted by ik...@apache.org.
Rename tsoclient package to tso.client
Change-Id: I989f8213f0a87513769b19ebdad5b5c89c679bcb
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/c6410f7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/c6410f7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/c6410f7d
Branch: refs/heads/master
Commit: c6410f7d06dd237e63b3324d6f2319667c88cca8
Parents: 516e517
Author: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Authored: Fri Apr 15 10:44:46 2016 -0700
Committer: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Committed: Fri Apr 15 10:44:46 2016 -0700
----------------------------------------------------------------------
.../yahoo/omid/benchmarks/tso/RawTxRunner.java | 10 +-
.../tso/TSOServerBenchmarkConfig.java | 2 +-
.../default-tso-server-benchmark-config.yml | 4 +-
.../omid/examples/ConfigurationExample.java | 2 +-
.../main/resources/hbase-omid-client-config.yml | 2 +-
.../transaction/HBaseAsyncPostCommitter.java | 2 +-
.../com/yahoo/omid/transaction/HBaseCellId.java | 2 +-
.../HBaseOmidClientConfiguration.java | 4 +-
.../transaction/HBaseSyncPostCommitter.java | 2 +-
.../transaction/HBaseTransactionManager.java | 4 +-
.../default-hbase-omid-client-config.yml | 2 +-
.../yahoo/omid/transaction/OmidTestBase.java | 4 +-
.../TestEndToEndScenariosWithHA.java | 2 +-
.../TestHBaseTransactionManager.java | 2 +-
.../yahoo/omid/transaction/TestShadowCells.java | 2 -
.../transaction/TestTransactionCleanup.java | 6 +-
.../omid/transaction/TestTxMgrFailover.java | 2 +-
.../resources/test-hbase-omid-client-config.yml | 4 +-
.../omid/transaction/AbstractTransaction.java | 2 +-
.../transaction/AbstractTransactionManager.java | 12 +-
.../omid/transaction/PostCommitActions.java | 2 +-
.../yahoo/omid/tso/client/AbortException.java | 27 +
.../java/com/yahoo/omid/tso/client/CellId.java | 24 +
.../yahoo/omid/tso/client/ClosingException.java | 28 +
.../omid/tso/client/ConnectionException.java | 30 +
.../omid/tso/client/ForwardingTSOFuture.java | 64 ++
.../tso/client/HandshakeFailedException.java | 28 +
.../yahoo/omid/tso/client/MockTSOClient.java | 102 ++
.../yahoo/omid/tso/client/NewTSOException.java | 27 +
.../tso/client/OmidClientConfiguration.java | 177 ++++
.../tso/client/ServiceUnavailableException.java | 32 +
.../com/yahoo/omid/tso/client/TSOClient.java | 933 +++++++++++++++++++
.../com/yahoo/omid/tso/client/TSOFuture.java | 25 +
.../com/yahoo/omid/tso/client/TSOProtocol.java | 56 ++
.../yahoo/omid/tso/util/DummyCellIdImpl.java | 2 +-
.../yahoo/omid/tsoclient/AbortException.java | 27 -
.../java/com/yahoo/omid/tsoclient/CellId.java | 24 -
.../yahoo/omid/tsoclient/ClosingException.java | 28 -
.../omid/tsoclient/ConnectionException.java | 30 -
.../omid/tsoclient/ForwardingTSOFuture.java | 64 --
.../tsoclient/HandshakeFailedException.java | 28 -
.../com/yahoo/omid/tsoclient/MockTSOClient.java | 102 --
.../yahoo/omid/tsoclient/NewTSOException.java | 27 -
.../omid/tsoclient/OmidClientConfiguration.java | 177 ----
.../tsoclient/ServiceUnavailableException.java | 32 -
.../com/yahoo/omid/tsoclient/TSOClient.java | 933 -------------------
.../com/yahoo/omid/tsoclient/TSOFuture.java | 25 -
.../com/yahoo/omid/tsoclient/TSOProtocol.java | 56 --
.../src/main/resources/omid-client-config.yml | 6 +-
.../omid/tso/client/TestMockTSOClient.java | 72 ++
.../tso/client/TestOmidClientConfiguration.java | 32 +
.../yahoo/omid/tsoclient/TestMockTSOClient.java | 72 --
.../tsoclient/TestOmidClientConfiguration.java | 32 -
.../com/yahoo/omid/tso/TestLeaseManager.java | 2 +-
.../omid/tso/client/TSOClientAccessor.java | 29 +
.../yahoo/omid/tso/client/TSOClientOneShot.java | 63 ++
.../com/yahoo/omid/tso/client/TSOClientRaw.java | 150 +++
...tionOfTSOClientServerBasicFunctionality.java | 233 +++++
.../client/TestTSOClientConnectionToTSO.java | 284 ++++++
...stTSOClientRequestAndResponseBehaviours.java | 423 +++++++++
.../client/TestTSOClientResponseHandling.java | 111 +++
.../tso/client/TestUnconnectedTSOClient.java | 83 ++
.../yahoo/omid/tsoclient/TSOClientAccessor.java | 30 -
.../yahoo/omid/tsoclient/TSOClientOneShot.java | 63 --
.../com/yahoo/omid/tsoclient/TSOClientRaw.java | 150 ---
...tionOfTSOClientServerBasicFunctionality.java | 233 -----
.../tsoclient/TestTSOClientConnectionToTSO.java | 285 ------
...stTSOClientRequestAndResponseBehaviours.java | 423 ---------
.../TestTSOClientResponseHandling.java | 111 ---
.../tsoclient/TestUnconnectedTSOClient.java | 84 --
70 files changed, 3074 insertions(+), 3079 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
index 4082b61..eaef6e7 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
@@ -29,11 +29,11 @@ import com.yahoo.omid.metrics.Counter;
import com.yahoo.omid.metrics.MetricsRegistry;
import com.yahoo.omid.metrics.Timer;
import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import com.yahoo.omid.tsoclient.AbortException;
-import com.yahoo.omid.tsoclient.CellId;
-import com.yahoo.omid.tsoclient.OmidClientConfiguration;
-import com.yahoo.omid.tsoclient.TSOClient;
-import com.yahoo.omid.tsoclient.TSOFuture;
+import com.yahoo.omid.tso.client.AbortException;
+import com.yahoo.omid.tso.client.CellId;
+import com.yahoo.omid.tso.client.OmidClientConfiguration;
+import com.yahoo.omid.tso.client.TSOClient;
+import com.yahoo.omid.tso.client.TSOFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
index ba42544..150040c 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
@@ -22,7 +22,7 @@ import com.yahoo.omid.YAMLUtils;
import com.yahoo.omid.benchmarks.utils.IntegerGenerator;
import com.yahoo.omid.metrics.MetricsRegistry;
import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import com.yahoo.omid.tsoclient.OmidClientConfiguration;
+import com.yahoo.omid.tso.client.OmidClientConfiguration;
public class TSOServerBenchmarkConfig extends SecureHBaseConfig {
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml b/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
index 0bce87e..5b07bf5 100644
--- a/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
+++ b/benchmarks/src/main/resources/default-tso-server-benchmark-config.yml
@@ -44,8 +44,8 @@ commitDelayInMs: 0
# ---------------------------------------------------------------------------------------------------------------------
# Omid TSO Client configuration
# ---------------------------------------------------------------------------------------------------------------------
-omidClientConfiguration: !!com.yahoo.omid.tsoclient.OmidClientConfiguration
- connectionType: !!com.yahoo.omid.tsoclient.OmidClientConfiguration$ConnType DIRECT
+omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
+ connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
connectionString: "localhost:54758"
# ---------------------------------------------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java b/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
index c2dc663..13efb50 100644
--- a/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
+++ b/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
@@ -32,7 +32,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
-import static com.yahoo.omid.tsoclient.OmidClientConfiguration.ConnType.DIRECT;
+import static com.yahoo.omid.tso.client.OmidClientConfiguration.ConnType.DIRECT;
/**
* ****************************************************************************************************************
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/examples/src/main/resources/hbase-omid-client-config.yml
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/hbase-omid-client-config.yml b/examples/src/main/resources/hbase-omid-client-config.yml
index 7236509..3314d5d 100644
--- a/examples/src/main/resources/hbase-omid-client-config.yml
+++ b/examples/src/main/resources/hbase-omid-client-config.yml
@@ -3,7 +3,7 @@
# principal: omid_hbase_client
# keytab: /path/to/hbase/client/keytab
-omidClientConfiguration: !!com.yahoo.omid.tsoclient.OmidClientConfiguration
+omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
connectionString: "localhost:54758"
metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
index 091cf58..8055a03 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
@@ -19,7 +19,7 @@ package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
-import com.yahoo.omid.tsoclient.CellId;
+import com.yahoo.omid.tso.client.CellId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
index c398fb4..ca9d675 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
@@ -18,7 +18,7 @@
package com.yahoo.omid.transaction;
import com.google.common.hash.Hashing;
-import com.yahoo.omid.tsoclient.CellId;
+import com.yahoo.omid.tso.client.CellId;
import org.apache.hadoop.hbase.client.HTableInterface;
import static com.google.common.base.Charsets.UTF_8;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
index fbc8119..19c90cb 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
@@ -23,8 +23,8 @@ import com.google.inject.name.Named;
import com.yahoo.omid.YAMLUtils;
import com.yahoo.omid.metrics.MetricsRegistry;
import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import com.yahoo.omid.tsoclient.OmidClientConfiguration.PostCommitMode;
-import com.yahoo.omid.tsoclient.OmidClientConfiguration;
+import com.yahoo.omid.tso.client.OmidClientConfiguration.PostCommitMode;
+import com.yahoo.omid.tso.client.OmidClientConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
index 0a90d68..0fc3cd1 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
@@ -22,7 +22,7 @@ import com.google.common.util.concurrent.SettableFuture;
import com.yahoo.omid.committable.CommitTable;
import com.yahoo.omid.metrics.MetricsRegistry;
import com.yahoo.omid.metrics.Timer;
-import com.yahoo.omid.tsoclient.CellId;
+import com.yahoo.omid.tso.client.CellId;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
index fe21cd1..941c583 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
@@ -28,8 +28,8 @@ import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
import com.yahoo.omid.committable.hbase.HBaseCommitTable;
import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
import com.yahoo.omid.tools.hbase.HBaseLogin;
-import com.yahoo.omid.tsoclient.CellId;
-import com.yahoo.omid.tsoclient.TSOClient;
+import com.yahoo.omid.tso.client.CellId;
+import com.yahoo.omid.tso.client.TSOClient;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/resources/default-hbase-omid-client-config.yml b/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
index 08d01d7..da59556 100644
--- a/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
+++ b/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
@@ -2,7 +2,7 @@
commitTableName: OMID_COMMIT_TABLE
#TSO/HA connection
-omidClientConfiguration: !!com.yahoo.omid.tsoclient.OmidClientConfiguration [ ]
+omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration [ ]
#Instrumentation
metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
index 61bf567..74063b2 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
@@ -28,8 +28,8 @@ import com.yahoo.omid.tools.hbase.OmidTableManager;
import com.yahoo.omid.tso.TSOMockModule;
import com.yahoo.omid.tso.TSOServer;
import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tsoclient.OmidClientConfiguration;
-import com.yahoo.omid.tsoclient.TSOClient;
+import com.yahoo.omid.tso.client.OmidClientConfiguration;
+import com.yahoo.omid.tso.client.TSOClient;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
index 8c060fb..0434520 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
@@ -49,7 +49,7 @@ import java.util.concurrent.TimeUnit;
import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME;
-import static com.yahoo.omid.tsoclient.OmidClientConfiguration.ConnType.HA;
+import static com.yahoo.omid.tso.client.OmidClientConfiguration.ConnType.HA;
import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
index beb999d..052f7da 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
@@ -17,7 +17,7 @@
*/
package com.yahoo.omid.transaction;
-import com.yahoo.omid.tsoclient.TSOClient;
+import com.yahoo.omid.tso.client.TSOClient;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
index 8d85dc5..7e9b406 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
@@ -23,8 +23,6 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.yahoo.omid.committable.CommitTable;
import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.tsoclient.TSOClient;
-import org.apache.commons.configuration.BaseConfiguration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
index 863480e..f74f659 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
@@ -18,9 +18,9 @@
package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.tsoclient.AbortException;
-import com.yahoo.omid.tsoclient.ForwardingTSOFuture;
-import com.yahoo.omid.tsoclient.TSOClient;
+import com.yahoo.omid.tso.client.AbortException;
+import com.yahoo.omid.tso.client.ForwardingTSOFuture;
+import com.yahoo.omid.tso.client.TSOClient;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
index bfa6b4d..f26efc6 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
@@ -25,7 +25,7 @@ import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
import com.yahoo.omid.committable.InMemoryCommitTable;
import com.yahoo.omid.transaction.Transaction.Status;
import com.yahoo.omid.tso.ProgrammableTSOServer;
-import com.yahoo.omid.tsoclient.TSOClient;
+import com.yahoo.omid.tso.client.TSOClient;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/resources/test-hbase-omid-client-config.yml b/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
index 86d1d66..9a95df3 100644
--- a/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
+++ b/hbase-client/src/test/resources/test-hbase-omid-client-config.yml
@@ -2,10 +2,10 @@
commitTableName: OMID_COMMIT_TABLE1
#TSO/HA connection
-omidClientConfiguration: !!com.yahoo.omid.tsoclient.OmidClientConfiguration
+omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
#TSO/HA connection
connectionString: "somehost:54758"
- connectionType: !!com.yahoo.omid.tsoclient.OmidClientConfiguration$ConnType HA
+ connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType HA
zkConnectionTimeoutInSecs: 11
#TSO related
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
index d303a04..f09772a 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
@@ -18,7 +18,7 @@
package com.yahoo.omid.transaction;
import com.google.common.base.Optional;
-import com.yahoo.omid.tsoclient.CellId;
+import com.yahoo.omid.tso.client.CellId;
import java.util.ArrayList;
import java.util.HashMap;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
index 83c766a..42bec31 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
@@ -26,12 +26,12 @@ import com.yahoo.omid.metrics.Counter;
import com.yahoo.omid.metrics.MetricsRegistry;
import com.yahoo.omid.metrics.Timer;
import com.yahoo.omid.transaction.Transaction.Status;
-import com.yahoo.omid.tsoclient.AbortException;
-import com.yahoo.omid.tsoclient.CellId;
-import com.yahoo.omid.tsoclient.ConnectionException;
-import com.yahoo.omid.tsoclient.NewTSOException;
-import com.yahoo.omid.tsoclient.ServiceUnavailableException;
-import com.yahoo.omid.tsoclient.TSOClient;
+import com.yahoo.omid.tso.client.AbortException;
+import com.yahoo.omid.tso.client.CellId;
+import com.yahoo.omid.tso.client.ConnectionException;
+import com.yahoo.omid.tso.client.NewTSOException;
+import com.yahoo.omid.tso.client.ServiceUnavailableException;
+import com.yahoo.omid.tso.client.TSOClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
index 30b51f2..6b7b2ba 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
@@ -18,7 +18,7 @@
package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.ListenableFuture;
-import com.yahoo.omid.tsoclient.CellId;
+import com.yahoo.omid.tso.client.CellId;
public interface PostCommitActions {
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java
new file mode 100644
index 0000000..670e87f
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+/**
+ * Thrown when the TSO server has aborted a transaction
+ */
+public class AbortException extends Exception {
+
+ private static final long serialVersionUID = 1861474360100681040L;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java
new file mode 100644
index 0000000..d50aac3
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/CellId.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+public interface CellId {
+
+ public long getCellId();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java
new file mode 100644
index 0000000..5c9cd23
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ClosingException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+/**
+ * Thrown when an error is produced when performing the actions required
+ * to close the communication with the TSO server
+ */
+public class ClosingException extends Exception {
+
+ private static final long serialVersionUID = -5681694952053689884L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java
new file mode 100644
index 0000000..e7ba0a0
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ConnectionException.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import java.io.IOException;
+
+/**
+ * Thrown when there are problems with the comm channel with the TSO server
+ * (e.g. when it is closed or disconnected)
+ */
+public class ConnectionException extends IOException {
+
+ private static final long serialVersionUID = -8489539195700267443L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java
new file mode 100644
index 0000000..8a7089c
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ForwardingTSOFuture.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class ForwardingTSOFuture<T> implements TSOFuture<T> {
+ private final ListenableFuture<T> future;
+
+ public ForwardingTSOFuture(ListenableFuture<T> future) {
+ this.future = future;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return future.cancel(mayInterruptIfRunning);
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return future.isCancelled();
+ }
+
+ @Override
+ public boolean isDone() {
+ return future.isDone();
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ return future.get();
+ }
+
+ @Override
+ public T get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return future.get(timeout, unit);
+ }
+
+ @Override
+ public void addListener(Runnable listener, Executor executor) {
+ future.addListener(listener, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java
new file mode 100644
index 0000000..6d2764a
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/HandshakeFailedException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+/**
+ * Thrown when some incompatibilities between the TSO client & server are
+ * found
+ */
+public class HandshakeFailedException extends Exception {
+
+ private static final long serialVersionUID = 8545505066920548834L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java
new file mode 100644
index 0000000..5b67441
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/MockTSOClient.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.util.concurrent.SettableFuture;
+import com.yahoo.omid.committable.CommitTable;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MockTSOClient implements TSOProtocol {
+ private final AtomicLong timestampGenerator = new AtomicLong();
+ private final int CONFLICT_MAP_SIZE = 1 * 1000 * 1000;
+ private final long[] conflictMap = new long[CONFLICT_MAP_SIZE];
+ private final AtomicLong lwm = new AtomicLong();
+
+ private final CommitTable.Writer commitTable;
+
+ public MockTSOClient(CommitTable.Writer commitTable) {
+ this.commitTable = commitTable;
+ }
+
+ @Override
+ public TSOFuture<Long> getNewStartTimestamp() {
+ synchronized (conflictMap) {
+ SettableFuture<Long> f = SettableFuture.<Long>create();
+ f.set(timestampGenerator.incrementAndGet());
+ return new ForwardingTSOFuture<Long>(f);
+ }
+ }
+
+ @Override
+ public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
+ synchronized (conflictMap) {
+ SettableFuture<Long> f = SettableFuture.<Long>create();
+ if (transactionId < lwm.get()) {
+ f.setException(new AbortException());
+ return new ForwardingTSOFuture<Long>(f);
+ }
+
+ boolean canCommit = true;
+ for (CellId c : cells) {
+ int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
+ if (conflictMap[index] >= transactionId) {
+ canCommit = false;
+ break;
+ }
+ }
+
+ if (canCommit) {
+ long commitTimestamp = timestampGenerator.incrementAndGet();
+ for (CellId c : cells) {
+ int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
+ long oldVal = conflictMap[index];
+ conflictMap[index] = commitTimestamp;
+ long curLwm = lwm.get();
+ while (oldVal > curLwm) {
+ if (lwm.compareAndSet(curLwm, oldVal)) {
+ break;
+ }
+ curLwm = lwm.get();
+ }
+ }
+
+ f.set(commitTimestamp);
+ try {
+ commitTable.addCommittedTransaction(transactionId, commitTimestamp);
+ commitTable.updateLowWatermark(lwm.get());
+ commitTable.flush();
+ } catch (IOException ioe) {
+ f.setException(ioe);
+ }
+ } else {
+ f.setException(new AbortException());
+ }
+ return new ForwardingTSOFuture<Long>(f);
+ }
+ }
+
+ @Override
+ public TSOFuture<Void> close() {
+ SettableFuture<Void> f = SettableFuture.<Void>create();
+ f.set(null);
+ return new ForwardingTSOFuture<Void>(f);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java
new file mode 100644
index 0000000..dac86e3
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/NewTSOException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+/**
+ * Thrown when a new TSO has been detected
+ */
+public class NewTSOException extends Exception {
+
+ private static final long serialVersionUID = -3250655858200759321L;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java
new file mode 100644
index 0000000..ad37889
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/OmidClientConfiguration.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import com.yahoo.omid.YAMLUtils;
+
+/**
+ * Configuration for Omid client side
+ */
+public class OmidClientConfiguration {
+
+ private static final String DEFAULT_CONFIG_FILE_NAME = "omid-client-config.yml";
+
+ public enum ConnType {DIRECT, HA}
+
+ public enum PostCommitMode {SYNC, ASYNC}
+
+ // Basic connection related params
+
+ private ConnType connectionType = ConnType.DIRECT;
+ private String connectionString;
+ private String zkCurrentTsoPath;
+ private String zkNamespace;
+ private int zkConnectionTimeoutInSecs;
+
+ // Communication protocol related params
+
+ private int requestMaxRetries;
+ private int requestTimeoutInMs;
+ private int reconnectionDelayInSecs;
+ private int retryDelayInMs;
+ private int executorThreads;
+
+ // Transaction Manager related params
+
+ private PostCommitMode postCommitMode = PostCommitMode.SYNC;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Instantiation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public OmidClientConfiguration() {
+ new YAMLUtils().loadSettings(DEFAULT_CONFIG_FILE_NAME, this);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters and setters for config params
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public ConnType getConnectionType() {
+ return connectionType;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.connectionType")
+ public void setConnectionType(ConnType connectionType) {
+ this.connectionType = connectionType;
+ }
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.connectionString")
+ public void setConnectionString(String connectionString) {
+ this.connectionString = connectionString;
+ }
+
+ public int getZkConnectionTimeoutInSecs() {
+ return zkConnectionTimeoutInSecs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.zkConnectionTimeoutInSecs")
+ public void setZkConnectionTimeoutInSecs(int zkConnectionTimeoutInSecs) {
+ this.zkConnectionTimeoutInSecs = zkConnectionTimeoutInSecs;
+ }
+
+ public int getRequestMaxRetries() {
+ return requestMaxRetries;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.requestMaxRetries")
+ public void setRequestMaxRetries(int requestMaxRetries) {
+ this.requestMaxRetries = requestMaxRetries;
+ }
+
+ public int getRequestTimeoutInMs() {
+ return requestTimeoutInMs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.requestTimeoutInMs")
+ public void setRequestTimeoutInMs(int requestTimeoutInMs) {
+ this.requestTimeoutInMs = requestTimeoutInMs;
+ }
+
+ public int getReconnectionDelayInSecs() {
+ return reconnectionDelayInSecs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.reconnectionDelayInSecs")
+ public void setReconnectionDelayInSecs(int reconnectionDelayInSecs) {
+ this.reconnectionDelayInSecs = reconnectionDelayInSecs;
+ }
+
+ public int getRetryDelayInMs() {
+ return retryDelayInMs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.retryDelayInMs")
+ public void setRetryDelayInMs(int retryDelayInMs) {
+ this.retryDelayInMs = retryDelayInMs;
+ }
+
+ public int getExecutorThreads() {
+ return executorThreads;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.executorThreads")
+ public void setExecutorThreads(int executorThreads) {
+ this.executorThreads = executorThreads;
+ }
+
+ public String getZkCurrentTsoPath() {
+ return zkCurrentTsoPath;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.ha.zkCurrentTsoPath")
+ public void setZkCurrentTsoPath(String zkCurrentTsoPath) {
+ this.zkCurrentTsoPath = zkCurrentTsoPath;
+ }
+
+ public String getZkNamespace() {
+ return zkNamespace;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.ha.zkNamespace")
+ public void setZkNamespace(String zkNamespace) {
+ this.zkNamespace = zkNamespace;
+ }
+
+ public PostCommitMode getPostCommitMode() {
+ return postCommitMode;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.tm.postCommitMode")
+ public void setPostCommitMode(PostCommitMode postCommitMode) {
+ this.postCommitMode = postCommitMode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java
new file mode 100644
index 0000000..4e0c9f7
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/ServiceUnavailableException.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+/**
+ * Thrown when the requests from TSO client to the TSO server have reached
+ * a number of retries
+ */
+public class ServiceUnavailableException extends Exception {
+
+ private static final long serialVersionUID = -1551974284011474385L;
+
+ public ServiceUnavailableException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java
new file mode 100644
index 0000000..dc2307c
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOClient.java
@@ -0,0 +1,933 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.base.Charsets;
+import com.google.common.net.HostAndPort;
+import com.google.common.util.concurrent.AbstractFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.yahoo.omid.proto.TSOProto;
+import com.yahoo.omid.zk.ZKUtils;
+import com.yahoo.statemachine.StateMachine;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.NodeCache;
+import org.apache.curator.framework.recipes.cache.NodeCacheListener;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
+import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.jboss.netty.util.Timeout;
+import org.jboss.netty.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayDeque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Describes the abstract methods to communicate to the TSO server
+ */
+public class TSOClient implements TSOProtocol, NodeCacheListener {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOClient.class);
+
+ // Basic configuration constants & defaults TODO: Move DEFAULT_ZK_CLUSTER to a conf class???
+ public static final String DEFAULT_ZK_CLUSTER = "localhost:2181";
+
+ private static final long DEFAULT_EPOCH = -1L;
+ private volatile long epoch = DEFAULT_EPOCH;
+
+ // Attributes
+ private CuratorFramework zkClient;
+ private NodeCache currentTSOZNode;
+
+ private ChannelFactory factory;
+ private ClientBootstrap bootstrap;
+ private Channel currentChannel;
+ private final ScheduledExecutorService fsmExecutor;
+ StateMachine.Fsm fsm;
+
+ private final int requestTimeoutInMs;
+ private final int requestMaxRetries;
+ private final int tsoReconnectionDelayInSecs;
+ private InetSocketAddress tsoAddr;
+ private String zkCurrentTsoPath;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Construction
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public static TSOClient newInstance(OmidClientConfiguration tsoClientConf)
+ throws IOException, InterruptedException {
+ return new TSOClient(tsoClientConf);
+ }
+
+ // Avoid instantiation
+ private TSOClient(OmidClientConfiguration omidConf) throws IOException, InterruptedException {
+
+ // Start client with Nb of active threads = 3 as maximum.
+ int tsoExecutorThreads = omidConf.getExecutorThreads();
+
+ factory = new NioClientSocketChannelFactory(
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), tsoExecutorThreads);
+ // Create the bootstrap
+ bootstrap = new ClientBootstrap(factory);
+
+ requestTimeoutInMs = omidConf.getRequestTimeoutInMs();
+ requestMaxRetries = omidConf.getRequestMaxRetries();
+ tsoReconnectionDelayInSecs = omidConf.getReconnectionDelayInSecs();
+
+ LOG.info("Connecting to TSO...");
+ HostAndPort hp;
+ switch (omidConf.getConnectionType()) {
+ case HA:
+ zkClient = ZKUtils.initZKClient(omidConf.getConnectionString(),
+ omidConf.getZkNamespace(),
+ omidConf.getZkConnectionTimeoutInSecs());
+ zkCurrentTsoPath = omidConf.getZkCurrentTsoPath();
+ configureCurrentTSOServerZNodeCache(zkCurrentTsoPath);
+ String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
+ // TSO info includes the new TSO host:port address and epoch
+ String[] currentTSOAndEpochArray = tsoInfo.split("#");
+ hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
+ setTSOAddress(hp.getHostText(), hp.getPort());
+ epoch = Long.parseLong(currentTSOAndEpochArray[1]);
+ LOG.info("\t* Current TSO host:port found in ZK: {} Epoch {}", hp, getEpoch());
+ break;
+ case DIRECT:
+ default:
+ hp = HostAndPort.fromString(omidConf.getConnectionString());
+ setTSOAddress(hp.getHostText(), hp.getPort());
+ LOG.info("\t* TSO host:port {} will be connected directly", hp);
+ break;
+ }
+
+ fsmExecutor = Executors.newSingleThreadScheduledExecutor(
+ new ThreadFactoryBuilder().setNameFormat("tsofsm-%d").build());
+ fsm = new StateMachine.FsmImpl(fsmExecutor);
+ fsm.setInitState(new DisconnectedState(fsm));
+
+ ChannelPipeline pipeline = bootstrap.getPipeline();
+ pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
+ pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+ pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
+ pipeline.addLast("protobufencoder", new ProtobufEncoder());
+ pipeline.addLast("handler", new Handler(fsm));
+
+ bootstrap.setOption("tcpNoDelay", true);
+ bootstrap.setOption("keepAlive", true);
+ bootstrap.setOption("reuseAddress", true);
+ bootstrap.setOption("connectTimeoutMillis", 100);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // TSOProtocol interface
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * @see TSOProtocol#getNewStartTimestamp()
+ */
+ @Override
+ public TSOFuture<Long> getNewStartTimestamp() {
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.TimestampRequest.Builder tsreqBuilder = TSOProto.TimestampRequest.newBuilder();
+ builder.setTimestampRequest(tsreqBuilder.build());
+ RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
+ fsm.sendEvent(request);
+ return new ForwardingTSOFuture<>(request);
+ }
+
+ /**
+ * @see TSOProtocol#commit(long, Set)
+ */
+ @Override
+ public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitbuilder = TSOProto.CommitRequest.newBuilder();
+ commitbuilder.setStartTimestamp(transactionId);
+ for (CellId cell : cells) {
+ commitbuilder.addCellId(cell.getCellId());
+ }
+ builder.setCommitRequest(commitbuilder.build());
+ RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
+ fsm.sendEvent(request);
+ return new ForwardingTSOFuture<>(request);
+ }
+
+ /**
+ * @see TSOProtocol#close()
+ */
+ @Override
+ public TSOFuture<Void> close() {
+ final CloseEvent closeEvent = new CloseEvent();
+ fsm.sendEvent(closeEvent);
+ closeEvent.addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ closeEvent.get();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ e.printStackTrace();
+ } catch (ExecutionException e) {
+ e.printStackTrace();
+ } finally {
+ fsmExecutor.shutdown();
+ if (currentTSOZNode != null) {
+ try {
+ currentTSOZNode.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ if (zkClient != null) {
+ zkClient.close();
+ }
+ }
+
+ }
+ }, fsmExecutor);
+ return new ForwardingTSOFuture<>(closeEvent);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // High availability related interface
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Returns the epoch of the TSO server that initialized this transaction.
+ * Used for high availability support.
+ */
+ public long getEpoch() {
+ return epoch;
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // NodeCacheListener interface
+ // ----------------------------------------------------------------------------------------------------------------
+
+ @Override
+ public void nodeChanged() throws Exception {
+
+ String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
+ // TSO info includes the new TSO host:port address and epoch
+ String[] currentTSOAndEpochArray = tsoInfo.split("#");
+ HostAndPort hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
+ setTSOAddress(hp.getHostText(), hp.getPort());
+ epoch = Long.parseLong(currentTSOAndEpochArray[1]);
+ LOG.info("CurrentTSO ZNode changed. New TSO Host & Port {}/Epoch {}", hp, getEpoch());
+ if (currentChannel != null && currentChannel.isConnected()) {
+ LOG.info("\tClosing channel with previous TSO {}", currentChannel);
+ currentChannel.close();
+ }
+
+ }
+
+ // ****************************************** Finite State Machine ************************************************
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // FSM: Events
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private static class ParamEvent<T> implements StateMachine.Event {
+
+ final T param;
+
+ ParamEvent(T param) {
+ this.param = param;
+ }
+
+ T getParam() {
+ return param;
+ }
+ }
+
+ private static class ErrorEvent extends ParamEvent<Throwable> {
+
+ ErrorEvent(Throwable t) {
+ super(t);
+ }
+ }
+
+ private static class ConnectedEvent extends ParamEvent<Channel> {
+
+ ConnectedEvent(Channel c) {
+ super(c);
+ }
+ }
+
+ private static class UserEvent<T> extends AbstractFuture<T>
+ implements StateMachine.DeferrableEvent {
+
+ void success(T value) {
+ set(value);
+ }
+
+ @Override
+ public void error(Throwable t) {
+ setException(t);
+ }
+ }
+
+ private static class CloseEvent extends UserEvent<Void> {
+
+ }
+
+ private static class ChannelClosedEvent extends ParamEvent<Throwable> {
+
+ ChannelClosedEvent(Throwable t) {
+ super(t);
+ }
+ }
+
+ private static class ReconnectEvent implements StateMachine.Event {
+
+ }
+
+ private static class HandshakeTimeoutEvent implements StateMachine.Event {
+
+ }
+
+ private static class TimestampRequestTimeoutEvent implements StateMachine.Event {
+
+ }
+
+ private static class CommitRequestTimeoutEvent implements StateMachine.Event {
+
+ final long startTimestamp;
+
+ CommitRequestTimeoutEvent(long startTimestamp) {
+ this.startTimestamp = startTimestamp;
+ }
+
+ public long getStartTimestamp() {
+ return startTimestamp;
+ }
+ }
+
+ private static class RequestEvent extends UserEvent<Long> {
+
+ TSOProto.Request req;
+ int retriesLeft;
+
+ RequestEvent(TSOProto.Request req, int retriesLeft) {
+ this.req = req;
+ this.retriesLeft = retriesLeft;
+ }
+
+ TSOProto.Request getRequest() {
+ return req;
+ }
+
+ void setRequest(TSOProto.Request request) {
+ this.req = request;
+ }
+
+ int getRetriesLeft() {
+ return retriesLeft;
+ }
+
+ void decrementRetries() {
+ retriesLeft--;
+ }
+
+ }
+
+ private static class ResponseEvent extends ParamEvent<TSOProto.Response> {
+
+ ResponseEvent(TSOProto.Response r) {
+ super(r);
+ }
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // FSM: States
+ // ----------------------------------------------------------------------------------------------------------------
+
+ class BaseState extends StateMachine.State {
+
+ BaseState(StateMachine.Fsm fsm) {
+ super(fsm);
+ }
+
+ public StateMachine.State handleEvent(StateMachine.Event e) {
+ LOG.error("Unhandled event {} while in state {}", e, this.getClass().getName());
+ return this;
+ }
+ }
+
+ class DisconnectedState extends BaseState {
+
+ DisconnectedState(StateMachine.Fsm fsm) {
+ super(fsm);
+ LOG.debug("NEW STATE: DISCONNECTED");
+ }
+
+ public StateMachine.State handleEvent(RequestEvent e) {
+ fsm.deferEvent(e);
+ return tryToConnectToTSOServer();
+ }
+
+ public StateMachine.State handleEvent(CloseEvent e) {
+ factory.releaseExternalResources();
+ e.success(null);
+ return this;
+ }
+
+ private StateMachine.State tryToConnectToTSOServer() {
+ final InetSocketAddress tsoAddress = getAddress();
+ LOG.info("Trying to connect to TSO [{}]", tsoAddress);
+ ChannelFuture channelFuture = bootstrap.connect(tsoAddress);
+ channelFuture.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture channelFuture) throws Exception {
+ if (channelFuture.isSuccess()) {
+ LOG.info("Connection to TSO [{}] established. Channel {}",
+ tsoAddress, channelFuture.getChannel());
+ } else {
+ LOG.error("Failed connection attempt to TSO [{}] failed. Channel {}",
+ tsoAddress, channelFuture.getChannel());
+ }
+ }
+ });
+ return new ConnectingState(fsm);
+ }
+ }
+
+ private class ConnectingState extends BaseState {
+
+ ConnectingState(StateMachine.Fsm fsm) {
+ super(fsm);
+ LOG.debug("NEW STATE: CONNECTING");
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ fsm.deferEvent(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ConnectedEvent e) {
+ return new HandshakingState(fsm, e.getParam());
+ }
+
+ public StateMachine.State handleEvent(ChannelClosedEvent e) {
+ return new ConnectionFailedState(fsm, e.getParam());
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ return new ConnectionFailedState(fsm, e.getParam());
+ }
+
+ }
+
+ private static class RequestAndTimeout {
+
+ final RequestEvent event;
+ final Timeout timeout;
+
+ RequestAndTimeout(RequestEvent event, Timeout timeout) {
+ this.event = event;
+ this.timeout = timeout;
+ }
+
+ RequestEvent getRequest() {
+ return event;
+ }
+
+ Timeout getTimeout() {
+ return timeout;
+ }
+
+ public String toString() {
+ String info = "Request type ";
+ if (event.getRequest().hasTimestampRequest()) {
+ info += "[Timestamp]";
+ } else if (event.getRequest().hasCommitRequest()) {
+ info += "[Commit] Start TS ->" + event.getRequest().getCommitRequest().getStartTimestamp();
+ } else {
+ info += "NONE";
+ }
+ return info;
+ }
+ }
+
+ private class HandshakingState extends BaseState {
+
+ final Channel channel;
+
+ final HashedWheelTimer timeoutExecutor = new HashedWheelTimer(
+ new ThreadFactoryBuilder().setNameFormat("tso-client-timeout").build());
+ final Timeout timeout;
+
+ HandshakingState(StateMachine.Fsm fsm, Channel channel) {
+ super(fsm);
+ LOG.debug("NEW STATE: HANDSHAKING");
+ this.channel = channel;
+ TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
+ // Add the required handshake capabilities when necessary
+ handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
+ channel.write(TSOProto.Request.newBuilder().setHandshakeRequest(handshake.build()).build());
+ timeout = newTimeout();
+ }
+
+ private Timeout newTimeout() {
+ if (requestTimeoutInMs > 0) {
+ return timeoutExecutor.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) {
+ fsm.sendEvent(new HandshakeTimeoutEvent());
+ }
+ }, 30, TimeUnit.SECONDS);
+ } else {
+ return null;
+ }
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ fsm.deferEvent(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ResponseEvent e) {
+ if (e.getParam().hasHandshakeResponse() && e.getParam().getHandshakeResponse().getClientCompatible()) {
+ if (timeout != null) {
+ timeout.cancel();
+ }
+ return new ConnectedState(fsm, channel, timeoutExecutor);
+ } else {
+ cleanupState();
+ LOG.error("Client incompatible with server");
+ return new HandshakeFailedState(fsm, new HandshakeFailedException());
+ }
+ }
+
+ public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
+ cleanupState();
+ return new ClosingState(fsm);
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ cleanupState();
+ Throwable exception = e.getParam();
+ LOG.error("Error during handshake", exception);
+ return new HandshakeFailedState(fsm, exception);
+ }
+
+ private void cleanupState() {
+ timeoutExecutor.stop();
+ channel.close();
+ if (timeout != null) {
+ timeout.cancel();
+ }
+ }
+
+ }
+
+ class ConnectionFailedState extends BaseState {
+
+ final HashedWheelTimer reconnectionTimeoutExecutor = new HashedWheelTimer(
+ new ThreadFactoryBuilder().setNameFormat("tso-client-backoff-timeout").build());
+
+ Throwable exception;
+
+ ConnectionFailedState(final StateMachine.Fsm fsm, final Throwable exception) {
+ super(fsm);
+ LOG.debug("NEW STATE: CONNECTION FAILED [RE-CONNECTION BACKOFF]");
+ this.exception = exception;
+ reconnectionTimeoutExecutor.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) {
+ fsm.sendEvent(new ReconnectEvent());
+ }
+ }, tsoReconnectionDelayInSecs, TimeUnit.SECONDS);
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ e.error(exception);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ChannelClosedEvent e) {
+ return new DisconnectedState(fsm);
+ }
+
+ public StateMachine.State handleEvent(ReconnectEvent e) {
+ return new DisconnectedState(fsm);
+ }
+
+ }
+
+ private class HandshakeFailedState extends ConnectionFailedState {
+
+ HandshakeFailedState(StateMachine.Fsm fsm, Throwable exception) {
+ super(fsm, exception);
+ LOG.debug("STATE: HANDSHAKING FAILED");
+ }
+
+ }
+
+ class ConnectedState extends BaseState {
+
+ final Queue<RequestAndTimeout> timestampRequests;
+ final Map<Long, RequestAndTimeout> commitRequests;
+ final Channel channel;
+
+ final HashedWheelTimer timeoutExecutor;
+
+ ConnectedState(StateMachine.Fsm fsm, Channel channel, HashedWheelTimer timeoutExecutor) {
+ super(fsm);
+ LOG.debug("NEW STATE: CONNECTED");
+ this.channel = channel;
+ this.timeoutExecutor = timeoutExecutor;
+ timestampRequests = new ArrayDeque<>();
+ commitRequests = new HashMap<>();
+ }
+
+ private Timeout newTimeout(final StateMachine.Event timeoutEvent) {
+ if (requestTimeoutInMs > 0) {
+ return timeoutExecutor.newTimeout(new TimerTask() {
+ @Override
+ public void run(Timeout timeout) {
+ fsm.sendEvent(timeoutEvent);
+ }
+ }, requestTimeoutInMs, TimeUnit.MILLISECONDS);
+ } else {
+ return null;
+ }
+ }
+
+ private void sendRequest(final StateMachine.Fsm fsm, RequestEvent request) {
+ TSOProto.Request req = request.getRequest();
+
+ if (req.hasTimestampRequest()) {
+ timestampRequests.add(new RequestAndTimeout(request, newTimeout(new TimestampRequestTimeoutEvent())));
+ } else if (req.hasCommitRequest()) {
+ TSOProto.CommitRequest commitReq = req.getCommitRequest();
+ commitRequests.put(commitReq.getStartTimestamp(), new RequestAndTimeout(
+ request, newTimeout(new CommitRequestTimeoutEvent(commitReq.getStartTimestamp()))));
+ } else {
+ request.error(new IllegalArgumentException("Unknown request type"));
+ return;
+ }
+ ChannelFuture f = channel.write(req);
+
+ f.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) {
+ if (!future.isSuccess()) {
+ fsm.sendEvent(new ErrorEvent(future.getCause()));
+ }
+ }
+ });
+ }
+
+ private void handleResponse(ResponseEvent response) {
+ TSOProto.Response resp = response.getParam();
+ if (resp.hasTimestampResponse()) {
+ if (timestampRequests.size() == 0) {
+ LOG.debug("Received timestamp response when no requests outstanding");
+ return;
+ }
+ RequestAndTimeout e = timestampRequests.remove();
+ e.getRequest().success(resp.getTimestampResponse().getStartTimestamp());
+ if (e.getTimeout() != null) {
+ e.getTimeout().cancel();
+ }
+ } else if (resp.hasCommitResponse()) {
+ long startTimestamp = resp.getCommitResponse().getStartTimestamp();
+ RequestAndTimeout e = commitRequests.remove(startTimestamp);
+ if (e == null) {
+ LOG.debug("Received commit response for request that doesn't exist. Start TS: {}", startTimestamp);
+ return;
+ }
+ if (e.getTimeout() != null) {
+ e.getTimeout().cancel();
+ }
+ if (resp.getCommitResponse().getAborted()) {
+ e.getRequest().error(new AbortException());
+ } else {
+ // Check if the commit response received implies heuristic
+ // actions during commit (because there's a new TSO master
+ // replica) and inform the caller (e.g. the TxMgr) about it
+ if (resp.getCommitResponse().getMakeHeuristicDecision()) {
+ e.getRequest().error(new NewTSOException());
+ } else {
+ e.getRequest().success(resp.getCommitResponse().getCommitTimestamp());
+ }
+ }
+ }
+ }
+
+ public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
+ if (!timestampRequests.isEmpty()) {
+ RequestAndTimeout r = timestampRequests.remove();
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ }
+ return this;
+ }
+
+ public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
+ long startTimestamp = e.getStartTimestamp();
+ if (commitRequests.containsKey(startTimestamp)) {
+ RequestAndTimeout r = commitRequests.remove(startTimestamp);
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ }
+ return this;
+ }
+
+ public StateMachine.State handleEvent(CloseEvent e) {
+ LOG.debug("CONNECTED STATE: CloseEvent");
+ timeoutExecutor.stop();
+ closeChannelAndErrorRequests();
+ fsm.deferEvent(e);
+ return new ClosingState(fsm);
+ }
+
+ public StateMachine.State handleEvent(RequestEvent e) {
+ sendRequest(fsm, e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ResponseEvent e) {
+ handleResponse(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ LOG.debug("CONNECTED STATE: ErrorEvent");
+ timeoutExecutor.stop();
+ handleError(fsm);
+ return new ClosingState(fsm);
+ }
+
+ private void handleError(StateMachine.Fsm fsm) {
+ LOG.debug("CONNECTED STATE: Cancelling Timeouts in handleError");
+ while (timestampRequests.size() > 0) {
+ RequestAndTimeout r = timestampRequests.remove();
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ }
+ Iterator<Map.Entry<Long, RequestAndTimeout>> iter = commitRequests.entrySet().iterator();
+ while (iter.hasNext()) {
+ RequestAndTimeout r = iter.next().getValue();
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ queueRetryOrError(fsm, r.getRequest());
+ iter.remove();
+ }
+ channel.close();
+ }
+
+ private void queueRetryOrError(StateMachine.Fsm fsm, RequestEvent e) {
+ if (e.getRetriesLeft() > 0) {
+ e.decrementRetries();
+ if (e.getRequest().hasCommitRequest()) {
+ TSOProto.CommitRequest commitRequest = e.getRequest().getCommitRequest();
+ if (!commitRequest.getIsRetry()) { // Create a new retry for the commit request
+ TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
+ TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
+ commitBuilder.mergeFrom(commitRequest);
+ commitBuilder.setIsRetry(true);
+ builder.setCommitRequest(commitBuilder.build());
+ e.setRequest(builder.build());
+ }
+ }
+ fsm.sendEvent(e);
+ } else {
+ e.error(
+ new ServiceUnavailableException("Number of retries exceeded. This API request failed permanently"));
+ }
+ }
+
+ private void closeChannelAndErrorRequests() {
+ channel.close();
+ for (RequestAndTimeout r : timestampRequests) {
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ r.getRequest().error(new ClosingException());
+ }
+ for (RequestAndTimeout r : commitRequests.values()) {
+ if (r.getTimeout() != null) {
+ r.getTimeout().cancel();
+ }
+ r.getRequest().error(new ClosingException());
+ }
+ }
+ }
+
+ private class ClosingState extends BaseState {
+
+ ClosingState(StateMachine.Fsm fsm) {
+ super(fsm);
+ LOG.debug("NEW STATE: CLOSING");
+ }
+
+ public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ErrorEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ResponseEvent e) {
+ // Ignored. They will be retried or errored
+ return this;
+ }
+
+ public StateMachine.State handleEvent(UserEvent e) {
+ fsm.deferEvent(e);
+ return this;
+ }
+
+ public StateMachine.State handleEvent(ChannelClosedEvent e) {
+ return new DisconnectedState(fsm);
+ }
+
+ public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
+ return this;
+ }
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper classes & methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private class Handler extends SimpleChannelHandler {
+
+ private StateMachine.Fsm fsm;
+
+ Handler(StateMachine.Fsm fsm) {
+ this.fsm = fsm;
+ }
+
+ @Override
+ public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
+ currentChannel = e.getChannel();
+ LOG.debug("HANDLER (CHANNEL CONNECTED): Connection {}. Sending connected event to FSM", e);
+ fsm.sendEvent(new ConnectedEvent(e.getChannel()));
+ }
+
+ @Override
+ public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ LOG.debug("HANDLER (CHANNEL DISCONNECTED): Connection {}. Sending error event to FSM", e);
+ fsm.sendEvent(new ErrorEvent(new ConnectionException()));
+ }
+
+ @Override
+ public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
+ LOG.debug("HANDLER (CHANNEL CLOSED): Connection {}. Sending channel closed event to FSM", e);
+ fsm.sendEvent(new ChannelClosedEvent(new ConnectionException()));
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
+ if (e.getMessage() instanceof TSOProto.Response) {
+ fsm.sendEvent(new ResponseEvent((TSOProto.Response) e.getMessage()));
+ } else {
+ LOG.warn("Received unknown message", e.getMessage());
+ }
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+ LOG.error("Error on channel {}", ctx.getChannel(), e.getCause());
+ fsm.sendEvent(new ErrorEvent(e.getCause()));
+ }
+ }
+
+ private synchronized void setTSOAddress(String host, int port) {
+ tsoAddr = new InetSocketAddress(host, port);
+ }
+
+ private synchronized InetSocketAddress getAddress() {
+ return tsoAddr;
+ }
+
+ private void configureCurrentTSOServerZNodeCache(String currentTsoPath) {
+ try {
+ currentTSOZNode = new NodeCache(zkClient, currentTsoPath);
+ currentTSOZNode.getListenable().addListener(this);
+ currentTSOZNode.start(true);
+ } catch (Exception e) {
+ throw new IllegalStateException("Cannot start watcher on current TSO Server ZNode: " + e.getMessage());
+ }
+ }
+
+ private String getCurrentTSOInfoFoundInZK(String currentTsoPath) {
+ ChildData currentTSOData = currentTSOZNode.getCurrentData();
+ if (currentTSOData == null) {
+ throw new IllegalStateException("No data found in ZKNode " + currentTsoPath);
+ }
+ byte[] currentTSOAndEpochAsBytes = currentTSOData.getData();
+ if (currentTSOAndEpochAsBytes == null) {
+ throw new IllegalStateException("No data found for current TSO in ZKNode " + currentTsoPath);
+ }
+ return new String(currentTSOAndEpochAsBytes, Charsets.UTF_8);
+ }
+
+}
[13/52] [abbrv] incubator-omid git commit: [ci skip]prepare release
omid-0.8.1.36
Posted by ik...@apache.org.
[ci skip]prepare release omid-0.8.1.36
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/6dd07aff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/6dd07aff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/6dd07aff
Branch: refs/heads/master
Commit: 6dd07aff59351ab777b3c9b8e6e7207d6cf4f924
Parents: b0d7034
Author: Omid CI <om...@yahoo-inc.com>
Authored: Fri Apr 15 19:09:31 2016 +0000
Committer: Omid CI <om...@yahoo-inc.com>
Committed: Fri Apr 15 19:09:31 2016 +0000
----------------------------------------------------------------------
benchmarks/pom.xml | 2 +-
codahale-metrics/pom.xml | 2 +-
commit-table/pom.xml | 2 +-
common/pom.xml | 2 +-
examples/pom.xml | 2 +-
hbase-client/pom.xml | 2 +-
hbase-commit-table/pom.xml | 2 +-
hbase-common/pom.xml | 2 +-
hbase-coprocessor/pom.xml | 2 +-
hbase-shims/hbase-0/pom.xml | 2 +-
hbase-shims/hbase-1/pom.xml | 2 +-
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 2 +-
metrics/pom.xml | 2 +-
pom.xml | 4 ++--
statemachine/pom.xml | 2 +-
timestamp-storage/pom.xml | 2 +-
transaction-client/pom.xml | 2 +-
tso-server/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index d2366f4..c5f6c01 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>benchmarks</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index 510778c..98656d6 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index 872dda8..9dd06a0 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 67ab4ef..8e6d498 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index da4d918..737f89a 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 9b5a7ca..28f5268 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index 3fe214d..8ebac20 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase-commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 2c307f5..380a144 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index 6781a50..e7b2948 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase-coprocessor</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index 8f1f530..e4449df 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase0-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index 7539e77..6f58a75 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase1-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index e1ec152..98eecab 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>omid-shims-aggregator</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index e1d6ab8..9fa8235 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>hbase-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index dd5c642..1b097a9 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>metrics</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b3b6af5..b93e769 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
<licenses>
<license>
@@ -46,7 +46,7 @@
<connection>scm:git:git://github.com/yahoo/omid.git</connection>
<developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
<url>https://github.com/yahoo/omid</url>
- <tag>omid-0.8.1.21</tag>
+ <tag>omid-0.8.1.36</tag>
</scm>
<distributionManagement>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index 717886f..22f76b3 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<name>State Machine</name>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index 6085da0..c413f43 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>timestamp-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index 2531c5d..7e4e8ff 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>transaction-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/6dd07aff/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index 32d6cef..c06345a 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36-SNAPSHOT</version>
+ <version>0.8.1.36</version>
</parent>
<artifactId>tso-server</artifactId>
[26/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorage.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorage.java
new file mode 100644
index 0000000..02bd5f6
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorage.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.recipes.atomic.AtomicValue;
+import org.apache.curator.framework.recipes.atomic.DistributedAtomicLong;
+import org.apache.curator.retry.RetryNTimes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.io.IOException;
+
+import static org.apache.omid.timestamp.storage.ZKTimestampPaths.TIMESTAMP_ZNODE;
+
+class ZKTimestampStorage implements TimestampStorage {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKTimestampStorage.class);
+
+ static final long INITIAL_MAX_TS_VALUE = 0;
+
+ private final DistributedAtomicLong timestamp;
+
+ @Inject
+ public ZKTimestampStorage(CuratorFramework zkClient) throws Exception {
+ LOG.info("ZK Client state {}", zkClient.getState());
+ timestamp = new DistributedAtomicLong(zkClient, TIMESTAMP_ZNODE, new RetryNTimes(3, 1000)); // TODO Configure
+ // this?
+ if (timestamp.initialize(INITIAL_MAX_TS_VALUE)) {
+ LOG.info("Timestamp value in ZNode initialized to {}", INITIAL_MAX_TS_VALUE);
+ }
+ }
+
+ @Override
+ public void updateMaxTimestamp(long previousMaxTimestamp, long newMaxTimestamp) throws IOException {
+
+ if (newMaxTimestamp < 0) {
+ LOG.error("Negative value received for maxTimestamp: {}", newMaxTimestamp);
+ throw new IllegalArgumentException();
+ }
+ if (newMaxTimestamp <= previousMaxTimestamp) {
+ LOG.error("maxTimestamp {} <= previousMaxTimesamp: {}", newMaxTimestamp, previousMaxTimestamp);
+ throw new IllegalArgumentException();
+ }
+ AtomicValue<Long> compareAndSet;
+ try {
+ compareAndSet = timestamp.compareAndSet(previousMaxTimestamp, newMaxTimestamp);
+ } catch (Exception e) {
+ throw new IOException("Problem setting timestamp in ZK", e);
+ }
+ if (!compareAndSet.succeeded()) { // We have to explicitly check for success (See Curator doc)
+ throw new IOException("GetAndSet operation for storing timestamp in ZK did not succeed "
+ + compareAndSet.preValue() + " " + compareAndSet.postValue());
+ }
+
+ }
+
+ @Override
+ public long getMaxTimestamp() throws IOException {
+
+ AtomicValue<Long> atomicValue;
+ try {
+ atomicValue = timestamp.get();
+ } catch (Exception e) {
+ throw new IOException("Problem getting data from ZK", e);
+ }
+ if (!atomicValue.succeeded()) { // We have to explicitly check for success (See Curator doc)
+ throw new IOException("Get operation to obtain timestamp from ZK did not succeed");
+ }
+ return atomicValue.postValue();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorageModule.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorageModule.java b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorageModule.java
new file mode 100644
index 0000000..48e46c5
--- /dev/null
+++ b/timestamp-storage/src/main/java/org/apache/omid/timestamp/storage/ZKTimestampStorageModule.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import com.google.inject.AbstractModule;
+import org.apache.curator.framework.CuratorFramework;
+
+import javax.inject.Singleton;
+
+public class ZKTimestampStorageModule extends AbstractModule {
+
+
+ @Override
+ public void configure() {
+
+ requireBinding(CuratorFramework.class);
+ // Timestamp storage creation
+ bind(TimestampStorage.class).to(ZKTimestampStorage.class).in(Singleton.class);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java b/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
deleted file mode 100644
index c273995..0000000
--- a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestHBaseTimestampStorage.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-
-import static com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig.*;
-import static org.testng.AssertJUnit.assertEquals;
-
-public class TestHBaseTimestampStorage {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTimestampStorage.class);
-
- private static final TableName TABLE_NAME = TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
-
- private static HBaseTestingUtility testutil;
- private static MiniHBaseCluster hbasecluster;
- protected static Configuration hbaseConf;
-
-
- @BeforeClass
- public static void setUpClass() throws Exception {
- // HBase setup
- hbaseConf = HBaseConfiguration.create();
-
- LOG.info("Create hbase");
- testutil = new HBaseTestingUtility(hbaseConf);
- hbasecluster = testutil.startMiniCluster(1);
-
- }
-
- @AfterClass
- public static void tearDownClass() throws Exception {
- if (hbasecluster != null) {
- testutil.shutdownMiniCluster();
- }
- }
-
- @BeforeMethod
- public void setUp() throws Exception {
- HBaseAdmin admin = testutil.getHBaseAdmin();
-
- if (!admin.tableExists(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME)) {
- HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
- HColumnDescriptor datafam = new HColumnDescriptor(DEFAULT_TIMESTAMP_STORAGE_CF_NAME);
- datafam.setMaxVersions(Integer.MAX_VALUE);
- desc.addFamily(datafam);
-
- admin.createTable(desc);
- }
-
- if (admin.isTableDisabled(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME)) {
- admin.enableTable(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
- }
- HTableDescriptor[] tables = admin.listTables();
- for (HTableDescriptor t : tables) {
- LOG.info(t.getNameAsString());
- }
- }
-
- @AfterMethod
- public void tearDown() {
- try {
- LOG.info("tearing Down");
- HBaseAdmin admin = testutil.getHBaseAdmin();
- admin.disableTable(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
- admin.deleteTable(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
-
- } catch (Exception e) {
- LOG.error("Error tearing down", e);
- }
- }
-
- @Test
- public void testHBaseTimestampStorage() throws Exception {
-
- final long INITIAL_TS_VALUE = 0;
- HBaseTimestampStorageConfig config = new HBaseTimestampStorageConfig();
- HBaseTimestampStorage tsStorage = new HBaseTimestampStorage(hbaseConf, config);
-
- // Test that the first time we get the timestamp is the initial value
- assertEquals("Initial value should be " + INITIAL_TS_VALUE, INITIAL_TS_VALUE, tsStorage.getMaxTimestamp());
-
- // Test that updating the timestamp succeeds when passing the initial value as the previous one
- long newTimestamp = 1;
- tsStorage.updateMaxTimestamp(INITIAL_TS_VALUE, newTimestamp);
-
- // Test setting a new timestamp fails (exception is thrown) when passing a wrong previous max timestamp
- long wrongTimestamp = 20;
- try {
- tsStorage.updateMaxTimestamp(wrongTimestamp, newTimestamp);
- Assert.fail("Shouldn't update");
- } catch (IOException e) {
- // Correct behavior
- }
- assertEquals("Value should be still " + newTimestamp, newTimestamp, tsStorage.getMaxTimestamp());
-
- // Test we can set a new timestamp when passing the right previous max timestamp
- long veryNewTimestamp = 40;
- tsStorage.updateMaxTimestamp(newTimestamp, veryNewTimestamp);
- assertEquals("Value should be " + veryNewTimestamp, veryNewTimestamp, tsStorage.getMaxTimestamp());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java b/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
deleted file mode 100644
index 363768f..0000000
--- a/timestamp-storage/src/test/java/com/yahoo/omid/timestamp/storage/TestZKTimestampStorage.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.timestamp.storage;
-
-import org.apache.curator.RetryPolicy;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.ExponentialBackoffRetry;
-import org.apache.curator.test.TestingServer;
-import org.apache.curator.utils.CloseableUtils;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.data.Stat;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.net.BindException;
-
-import static com.yahoo.omid.timestamp.storage.ZKTimestampPaths.TIMESTAMP_ZNODE;
-import static com.yahoo.omid.timestamp.storage.ZKTimestampStorage.INITIAL_MAX_TS_VALUE;
-import static org.mockito.Mockito.doThrow;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-public class TestZKTimestampStorage {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestZKTimestampStorage.class);
-
- private static final int BYTES_IN_LONG = 8;
-
- private static final int ZK_PORT = 16666;
- private static final String ZK_CLUSTER = "localhost:" + ZK_PORT;
-
- private static final long NEGATIVE_TS = -1;
-
- private static final int ITERATION_COUNT = 10;
-
- private TestingServer zkServer;
-
- private CuratorFramework zkClient;
-
- private ZKTimestampStorage storage;
-
- private CuratorFramework storageInternalZKClient;
-
- @BeforeMethod
- public void initStuff() throws Exception {
- LOG.info("Creating ZK server instance listening in port {}...", ZK_PORT);
- while (zkServer == null) {
- try {
- zkServer = new TestingServer(ZK_PORT);
- } catch (BindException e) {
- System.err.println("Getting bind exception - retrying to allocate server");
- zkServer = null;
- }
- }
- LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
-
- RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
-
- LOG.info("Creating Zookeeper Client connected to {}", ZK_CLUSTER);
- zkClient = CuratorFrameworkFactory.builder()
- .namespace("omid")
- .connectString(ZK_CLUSTER)
- .retryPolicy(retryPolicy)
- .connectionTimeoutMs(10) // Low timeout for tests
- .build();
- zkClient.start();
- zkClient.blockUntilConnected();
-
- LOG.info("Creating Internal Zookeeper Client connected to {}", ZK_CLUSTER);
- storageInternalZKClient = Mockito.spy(CuratorFrameworkFactory.builder()
- .namespace("omid")
- .connectString(ZK_CLUSTER)
- .retryPolicy(retryPolicy)
- .connectionTimeoutMs(10) // Low timeout for tests
- .build());
- storageInternalZKClient.start();
- storageInternalZKClient.blockUntilConnected();
-
- storage = new ZKTimestampStorage(storageInternalZKClient);
- }
-
- @AfterMethod
- public void closeStuff() throws Exception {
-
- CloseableUtils.closeQuietly(zkClient);
- LOG.info("ZK Client state {}", zkClient.getState());
- zkClient = null;
-
- CloseableUtils.closeQuietly(storageInternalZKClient);
- LOG.info("ZK Internal Client state {}", storageInternalZKClient.getState());
- storageInternalZKClient = null;
-
- CloseableUtils.closeQuietly(zkServer);
- LOG.info("ZK Server Stopped");
- zkServer = null;
-
- }
-
- @Test
- public void testBasicFunctionality() throws Exception {
-
- // Check ZNode for timestamp exists (storage instantiation should create it)
- Stat zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
- assertEquals(zNodeStats.getVersion(), 0);
-
- // Initial checks
- assertEquals(storage.getMaxTimestamp(), INITIAL_MAX_TS_VALUE);
- byte[] data = zkClient.getData().forPath(TIMESTAMP_ZNODE);
- assertEquals(data.length, BYTES_IN_LONG);
-
- // Check new timestamp does not allow negative values...
- try {
- storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, NEGATIVE_TS);
- fail();
- } catch (IllegalArgumentException e) {
- // Expected exception
- }
-
- // ...nor is less than previous timestamp
- try {
- storage.updateMaxTimestamp(1, 0);
- fail();
- } catch (IllegalArgumentException e) {
- // Expected exception
- }
-
- // Check that the original version is still there
- zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
- assertEquals(zNodeStats.getVersion(), 0);
-
- // Iterate updating the timestamp and check the final value
- long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
- for (int i = 0; i < ITERATION_COUNT; i++) {
- long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
- storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
- previousMaxTimestamp = newMaxTimestamp;
- }
- assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
- // Check the znode version has changed accordingly
- zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
- assertEquals(zNodeStats.getVersion(), ITERATION_COUNT);
-
- // Check exceptions
- doThrow(new RuntimeException()).when(storageInternalZKClient).getData();
- try {
- storage.getMaxTimestamp();
- fail();
- } catch (IOException e) {
- // Expected exception
- }
-
- doThrow(new RuntimeException()).when(storageInternalZKClient).setData();
- try {
- storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, INITIAL_MAX_TS_VALUE + 1_000_000);
- fail();
- } catch (IOException e) {
- // Expected exception
- }
-
- // Reset the mock and double-check last result
- Mockito.reset(storageInternalZKClient);
- assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
-
- // Finally check the znode version is still the same
- zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
- assertEquals(zNodeStats.getVersion(), ITERATION_COUNT);
- }
-
- @Test
- public void testZkClientWhenZKIsDownAndRestarts() throws Exception {
-
- // Iterate updating the timestamp and check the final value
- long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
- for (int i = 0; i < ITERATION_COUNT; i++) {
- long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
- storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
- previousMaxTimestamp = newMaxTimestamp;
- }
- assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
-
- // Stop ZK Server, expect the IO exception, reconnect and get the right value
- LOG.info("Stopping ZK Server");
- zkServer.stop();
- LOG.info("ZK Server Stopped");
-
- try {
- storage.getMaxTimestamp();
- fail();
- } catch (IOException ioe) {
- // Expected exception
- }
-
- LOG.info("Restarting ZK again");
- zkServer.restart();
- assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
-
- }
-
- @Test
- public void testZkClientLosingSession() throws Exception {
-
- // Cut the session in the server through the client
- long sessionId = zkClient.getZookeeperClient().getZooKeeper().getSessionId();
- byte[] sessionPasswd = zkClient.getZookeeperClient().getZooKeeper().getSessionPasswd();
- ZooKeeper zk = new ZooKeeper(ZK_CLUSTER, 1000, null, sessionId, sessionPasswd);
- zk.close();
- LOG.info("ZKClient session closed");
-
- // Iterate updating the timestamp and check the final value
- long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
- for (int i = 0; i < ITERATION_COUNT; i++) {
- long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
- storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
- LOG.info("Updating timestamp. Previous/New {}/{}", previousMaxTimestamp, newMaxTimestamp);
- previousMaxTimestamp = newMaxTimestamp;
- }
- assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestHBaseTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestHBaseTimestampStorage.java b/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestHBaseTimestampStorage.java
new file mode 100644
index 0000000..4218734
--- /dev/null
+++ b/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestHBaseTimestampStorage.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+import static org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig.*;
+import static org.testng.AssertJUnit.assertEquals;
+
+public class TestHBaseTimestampStorage {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestHBaseTimestampStorage.class);
+
+ private static final TableName TABLE_NAME = TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
+
+ private static HBaseTestingUtility testutil;
+ private static MiniHBaseCluster hbasecluster;
+ protected static Configuration hbaseConf;
+
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ // HBase setup
+ hbaseConf = HBaseConfiguration.create();
+
+ LOG.info("Create hbase");
+ testutil = new HBaseTestingUtility(hbaseConf);
+ hbasecluster = testutil.startMiniCluster(1);
+
+ }
+
+ @AfterClass
+ public static void tearDownClass() throws Exception {
+ if (hbasecluster != null) {
+ testutil.shutdownMiniCluster();
+ }
+ }
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ HBaseAdmin admin = testutil.getHBaseAdmin();
+
+ if (!admin.tableExists(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME)) {
+ HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+ HColumnDescriptor datafam = new HColumnDescriptor(DEFAULT_TIMESTAMP_STORAGE_CF_NAME);
+ datafam.setMaxVersions(Integer.MAX_VALUE);
+ desc.addFamily(datafam);
+
+ admin.createTable(desc);
+ }
+
+ if (admin.isTableDisabled(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME)) {
+ admin.enableTable(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
+ }
+ HTableDescriptor[] tables = admin.listTables();
+ for (HTableDescriptor t : tables) {
+ LOG.info(t.getNameAsString());
+ }
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ try {
+ LOG.info("tearing Down");
+ HBaseAdmin admin = testutil.getHBaseAdmin();
+ admin.disableTable(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
+ admin.deleteTable(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
+
+ } catch (Exception e) {
+ LOG.error("Error tearing down", e);
+ }
+ }
+
+ @Test
+ public void testHBaseTimestampStorage() throws Exception {
+
+ final long INITIAL_TS_VALUE = 0;
+ HBaseTimestampStorageConfig config = new HBaseTimestampStorageConfig();
+ HBaseTimestampStorage tsStorage = new HBaseTimestampStorage(hbaseConf, config);
+
+ // Test that the first time we get the timestamp is the initial value
+ assertEquals("Initial value should be " + INITIAL_TS_VALUE, INITIAL_TS_VALUE, tsStorage.getMaxTimestamp());
+
+ // Test that updating the timestamp succeeds when passing the initial value as the previous one
+ long newTimestamp = 1;
+ tsStorage.updateMaxTimestamp(INITIAL_TS_VALUE, newTimestamp);
+
+ // Test setting a new timestamp fails (exception is thrown) when passing a wrong previous max timestamp
+ long wrongTimestamp = 20;
+ try {
+ tsStorage.updateMaxTimestamp(wrongTimestamp, newTimestamp);
+ Assert.fail("Shouldn't update");
+ } catch (IOException e) {
+ // Correct behavior
+ }
+ assertEquals("Value should be still " + newTimestamp, newTimestamp, tsStorage.getMaxTimestamp());
+
+ // Test we can set a new timestamp when passing the right previous max timestamp
+ long veryNewTimestamp = 40;
+ tsStorage.updateMaxTimestamp(newTimestamp, veryNewTimestamp);
+ assertEquals("Value should be " + veryNewTimestamp, veryNewTimestamp, tsStorage.getMaxTimestamp());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestZKTimestampStorage.java
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestZKTimestampStorage.java b/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestZKTimestampStorage.java
new file mode 100644
index 0000000..1e370e2
--- /dev/null
+++ b/timestamp-storage/src/test/java/org/apache/omid/timestamp/storage/TestZKTimestampStorage.java
@@ -0,0 +1,241 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.timestamp.storage;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.curator.test.TestingServer;
+import org.apache.curator.utils.CloseableUtils;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.net.BindException;
+
+import static org.apache.omid.timestamp.storage.ZKTimestampPaths.TIMESTAMP_ZNODE;
+import static org.apache.omid.timestamp.storage.ZKTimestampStorage.INITIAL_MAX_TS_VALUE;
+import static org.mockito.Mockito.doThrow;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
+
+public class TestZKTimestampStorage {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestZKTimestampStorage.class);
+
+ private static final int BYTES_IN_LONG = 8;
+
+ private static final int ZK_PORT = 16666;
+ private static final String ZK_CLUSTER = "localhost:" + ZK_PORT;
+
+ private static final long NEGATIVE_TS = -1;
+
+ private static final int ITERATION_COUNT = 10;
+
+ private TestingServer zkServer;
+
+ private CuratorFramework zkClient;
+
+ private ZKTimestampStorage storage;
+
+ private CuratorFramework storageInternalZKClient;
+
+ @BeforeMethod
+ public void initStuff() throws Exception {
+ LOG.info("Creating ZK server instance listening in port {}...", ZK_PORT);
+ while (zkServer == null) {
+ try {
+ zkServer = new TestingServer(ZK_PORT);
+ } catch (BindException e) {
+ System.err.println("Getting bind exception - retrying to allocate server");
+ zkServer = null;
+ }
+ }
+ LOG.info("ZK Server Started @ {}", zkServer.getConnectString());
+
+ RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
+
+ LOG.info("Creating Zookeeper Client connected to {}", ZK_CLUSTER);
+ zkClient = CuratorFrameworkFactory.builder()
+ .namespace("omid")
+ .connectString(ZK_CLUSTER)
+ .retryPolicy(retryPolicy)
+ .connectionTimeoutMs(10) // Low timeout for tests
+ .build();
+ zkClient.start();
+ zkClient.blockUntilConnected();
+
+ LOG.info("Creating Internal Zookeeper Client connected to {}", ZK_CLUSTER);
+ storageInternalZKClient = Mockito.spy(CuratorFrameworkFactory.builder()
+ .namespace("omid")
+ .connectString(ZK_CLUSTER)
+ .retryPolicy(retryPolicy)
+ .connectionTimeoutMs(10) // Low timeout for tests
+ .build());
+ storageInternalZKClient.start();
+ storageInternalZKClient.blockUntilConnected();
+
+ storage = new ZKTimestampStorage(storageInternalZKClient);
+ }
+
+ @AfterMethod
+ public void closeStuff() throws Exception {
+
+ CloseableUtils.closeQuietly(zkClient);
+ LOG.info("ZK Client state {}", zkClient.getState());
+ zkClient = null;
+
+ CloseableUtils.closeQuietly(storageInternalZKClient);
+ LOG.info("ZK Internal Client state {}", storageInternalZKClient.getState());
+ storageInternalZKClient = null;
+
+ CloseableUtils.closeQuietly(zkServer);
+ LOG.info("ZK Server Stopped");
+ zkServer = null;
+
+ }
+
+ @Test
+ public void testBasicFunctionality() throws Exception {
+
+ // Check ZNode for timestamp exists (storage instantiation should create it)
+ Stat zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
+ assertEquals(zNodeStats.getVersion(), 0);
+
+ // Initial checks
+ assertEquals(storage.getMaxTimestamp(), INITIAL_MAX_TS_VALUE);
+ byte[] data = zkClient.getData().forPath(TIMESTAMP_ZNODE);
+ assertEquals(data.length, BYTES_IN_LONG);
+
+ // Check new timestamp does not allow negative values...
+ try {
+ storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, NEGATIVE_TS);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected exception
+ }
+
+ // ...nor is less than previous timestamp
+ try {
+ storage.updateMaxTimestamp(1, 0);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected exception
+ }
+
+ // Check that the original version is still there
+ zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
+ assertEquals(zNodeStats.getVersion(), 0);
+
+ // Iterate updating the timestamp and check the final value
+ long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
+ for (int i = 0; i < ITERATION_COUNT; i++) {
+ long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
+ storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
+ previousMaxTimestamp = newMaxTimestamp;
+ }
+ assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
+ // Check the znode version has changed accordingly
+ zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
+ assertEquals(zNodeStats.getVersion(), ITERATION_COUNT);
+
+ // Check exceptions
+ doThrow(new RuntimeException()).when(storageInternalZKClient).getData();
+ try {
+ storage.getMaxTimestamp();
+ fail();
+ } catch (IOException e) {
+ // Expected exception
+ }
+
+ doThrow(new RuntimeException()).when(storageInternalZKClient).setData();
+ try {
+ storage.updateMaxTimestamp(INITIAL_MAX_TS_VALUE, INITIAL_MAX_TS_VALUE + 1_000_000);
+ fail();
+ } catch (IOException e) {
+ // Expected exception
+ }
+
+ // Reset the mock and double-check last result
+ Mockito.reset(storageInternalZKClient);
+ assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
+
+ // Finally check the znode version is still the same
+ zNodeStats = zkClient.checkExists().forPath(TIMESTAMP_ZNODE);
+ assertEquals(zNodeStats.getVersion(), ITERATION_COUNT);
+ }
+
+ @Test
+ public void testZkClientWhenZKIsDownAndRestarts() throws Exception {
+
+ // Iterate updating the timestamp and check the final value
+ long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
+ for (int i = 0; i < ITERATION_COUNT; i++) {
+ long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
+ storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
+ previousMaxTimestamp = newMaxTimestamp;
+ }
+ assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
+
+ // Stop ZK Server, expect the IO exception, reconnect and get the right value
+ LOG.info("Stopping ZK Server");
+ zkServer.stop();
+ LOG.info("ZK Server Stopped");
+
+ try {
+ storage.getMaxTimestamp();
+ fail();
+ } catch (IOException ioe) {
+ // Expected exception
+ }
+
+ LOG.info("Restarting ZK again");
+ zkServer.restart();
+ assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
+
+ }
+
+ @Test
+ public void testZkClientLosingSession() throws Exception {
+
+ // Cut the session in the server through the client
+ long sessionId = zkClient.getZookeeperClient().getZooKeeper().getSessionId();
+ byte[] sessionPasswd = zkClient.getZookeeperClient().getZooKeeper().getSessionPasswd();
+ ZooKeeper zk = new ZooKeeper(ZK_CLUSTER, 1000, null, sessionId, sessionPasswd);
+ zk.close();
+ LOG.info("ZKClient session closed");
+
+ // Iterate updating the timestamp and check the final value
+ long previousMaxTimestamp = INITIAL_MAX_TS_VALUE;
+ for (int i = 0; i < ITERATION_COUNT; i++) {
+ long newMaxTimestamp = previousMaxTimestamp + 1_000_000;
+ storage.updateMaxTimestamp(previousMaxTimestamp, newMaxTimestamp);
+ LOG.info("Updating timestamp. Previous/New {}/{}", previousMaxTimestamp, newMaxTimestamp);
+ previousMaxTimestamp = newMaxTimestamp;
+ }
+ assertEquals(storage.getMaxTimestamp(), 1_000_000 * ITERATION_COUNT);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/timestamp-storage/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/timestamp-storage/src/test/resources/log4j.properties b/timestamp-storage/src/test/resources/log4j.properties
index 23b950e..3f565a2 100644
--- a/timestamp-storage/src/test/resources/log4j.properties
+++ b/timestamp-storage/src/test/resources/log4j.properties
@@ -39,14 +39,13 @@ log4j.logger.org.apache.zookeeper=ERROR
log4j.logger.org.apache.bookkeeper=FATAL
log4j.logger.org.apache.hadoop.hbase=ERROR
log4j.logger.org.apache.hadoop.ipc=ERROR
-
-log4j.logger.com.yahoo.omid.util=DEBUG
-#log4j.logger.com.yahoo.omid.regionserver.TransactionalRegionServer=TRACE
-#log4j.logger.com.yahoo.omid.TestBasicTransaction=TRACE
-#log4j.logger.com.yahoo.omid.client.TSOClient=TRACE
-#log4j.logger.com.yahoo.omid.client.TransactionState=TRACE
-#log4j.logger.com.yahoo.omid.OmidTestBase=TRACE
-#log4j.logger.com.yahoo.omid.tso.ThroughputMonitor=INFO
+log4j.logger.org.apache.omid.util=DEBUG
+#log4j.logger.org.apache.omid.regionserver.TransactionalRegionServer=TRACE
+#log4j.logger.org.apache.omid.TestBasicTransaction=TRACE
+#log4j.logger.org.apache.omid.client.TSOClient=TRACE
+#log4j.logger.org.apache.omid.client.TransactionState=TRACE
+#log4j.logger.org.apache.omid.OmidTestBase=TRACE
+#log4j.logger.org.apache.omid.tso.ThroughputMonitor=INFO
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
# Make these two classes INFO-level. Make them DEBUG to see more zk debug.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index f7c27d9..e0102a5 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -17,27 +17,27 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>statemachine</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>metrics</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>commit-table</artifactId>
<version>${project.version}</version>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
deleted file mode 100644
index f09772a..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransaction.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-import com.yahoo.omid.tso.client.CellId;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Omid's base abstract implementation of the {@link Transaction} interface.
- * Provides extra methods to access other basic transaction state required by
- * {@link TransactionManagerExtension} implementations based on snapshot
- * isolation.
- *
- * So, this abstract class must be extended by particular implementations of
- * transaction managers related to different storage systems (HBase...)
- */
-public abstract class AbstractTransaction<T extends CellId> implements Transaction {
-
- private transient Map<String, Object> metadata = new HashMap<>();
- private final AbstractTransactionManager transactionManager;
- private final long startTimestamp;
- private final long epoch;
- private long commitTimestamp;
- private boolean isRollbackOnly;
- private final Set<T> writeSet;
- private Status status = Status.RUNNING;
-
- /**
- * Base constructor
- *
- * @param transactionId
- * transaction identifier to assign
- * @param epoch
- * epoch of the TSOServer instance that created this transaction
- * Used in High Availability to guarantee data consistency
- * @param writeSet
- * initial write set for the transaction.
- * Should be empty in most cases.
- * @param transactionManager
- * transaction manager associated to this transaction.
- * Usually, should be the one that created the transaction
- * instance.
- */
- public AbstractTransaction(long transactionId,
- long epoch,
- Set<T> writeSet,
- AbstractTransactionManager transactionManager) {
- this.startTimestamp = transactionId;
- this.epoch = epoch;
- this.writeSet = writeSet;
- this.transactionManager = transactionManager;
- }
-
- /**
- * Allows to define specific clean-up task for transaction implementations
- */
- public abstract void cleanup();
-
- /**
- * @see com.yahoo.omid.transaction.Transaction#getTransactionId()
- */
- @Override
- public long getTransactionId() {
- return startTimestamp;
- }
-
- /**
- * @see com.yahoo.omid.transaction.Transaction#getEpoch()
- */
- @Override
- public long getEpoch() {
- return epoch;
- }
-
- /**
- * @see com.yahoo.omid.transaction.Transaction#getStatus()
- */
- @Override
- public Status getStatus() {
- return status;
- }
-
- /**
- * @see Transaction#isRollbackOnly()
- */
- @Override
- public void setRollbackOnly() {
- isRollbackOnly = true;
- }
-
- /**
- * @see com.yahoo.omid.transaction.Transaction#isRollbackOnly()
- */
- @Override
- public boolean isRollbackOnly() {
- return isRollbackOnly;
- }
-
- /**
- * Returns transaction manager associated to this transaction.
- * @return transaction manager
- */
- public AbstractTransactionManager getTransactionManager() {
- return transactionManager;
- }
-
- /**
- * Returns the start timestamp for this transaction.
- * @return start timestamp
- */
- public long getStartTimestamp() {
- return startTimestamp;
- }
-
- /**
- * Returns the commit timestamp for this transaction.
- * @return commit timestamp
- */
- public long getCommitTimestamp() {
- return commitTimestamp;
- }
-
- /**
- * Sets the commit timestamp for this transaction.
- * @param commitTimestamp
- * the commit timestamp to set
- */
- public void setCommitTimestamp(long commitTimestamp) {
- this.commitTimestamp = commitTimestamp;
- }
-
- /**
- * Sets the status for this transaction.
- * @param status
- * the {@link Status} to set
- */
- public void setStatus(Status status) {
- this.status = status;
- }
-
- /**
- * Returns the current write-set for this transaction.
- * @return write set
- */
- public Set<T> getWriteSet() {
- return writeSet;
- }
-
- /**
- * Adds an element to the transaction write-set.
- * @param element
- * the element to add
- */
- public void addWriteSetElement(T element) {
- writeSet.add(element);
- }
-
- @Override
- public String toString() {
- return String.format("Tx-%s [%s] (ST=%d, CT=%d, Epoch=%d) WriteSet %s",
- Long.toHexString(getTransactionId()),
- status,
- startTimestamp,
- commitTimestamp,
- epoch,
- writeSet);
- }
-
- @Override
- public Optional<Object> getMetadata(String key) {
- return Optional.fromNullable(metadata.get(key));
- }
-
- /**
- * Expects they metadata stored under key "key" to be of the "Set" type,
- * append "value" to the existing set or creates a new one
- */
- @Override
- @SuppressWarnings("unchecked")
- public void appendMetadata(String key, Object value) {
- List existingValue = (List) metadata.get(key);
- if (existingValue == null) {
- List<Object> newList = new ArrayList<>();
- newList.add(value);
- metadata.put(key, newList);
- } else {
- existingValue.add(value);
- }
- }
-
- @Override
- public void setMetadata(String key, Object value) {
- metadata.put(key, value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
deleted file mode 100644
index 42bec31..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/AbstractTransactionManager.java
+++ /dev/null
@@ -1,502 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.Futures;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.metrics.Counter;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.Timer;
-import com.yahoo.omid.transaction.Transaction.Status;
-import com.yahoo.omid.tso.client.AbortException;
-import com.yahoo.omid.tso.client.CellId;
-import com.yahoo.omid.tso.client.ConnectionException;
-import com.yahoo.omid.tso.client.NewTSOException;
-import com.yahoo.omid.tso.client.ServiceUnavailableException;
-import com.yahoo.omid.tso.client.TSOClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.CACHE;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.COMMIT_TABLE;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.NOT_PRESENT;
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.SHADOW_CELL;
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-
-/**
- * Omid's base abstract implementation of the {@link TransactionManager} interface.
- *
- * Provides extra methods to allow transaction manager developers to perform
- * different actions before/after the methods exposed by the {@link TransactionManager} interface.
- *
- * So, this abstract class must be extended by particular implementations of
- * transaction managers related to different storage systems (HBase...)
- */
-public abstract class AbstractTransactionManager implements TransactionManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionManager.class);
-
- public interface TransactionFactory<T extends CellId> {
-
- AbstractTransaction<T> createTransaction(long transactionId, long epoch, AbstractTransactionManager tm);
-
- }
-
- private final PostCommitActions postCommitter;
- protected final TSOClient tsoClient;
- protected final CommitTable.Client commitTableClient;
- private final TransactionFactory<? extends CellId> transactionFactory;
-
- // Metrics
- private final Timer startTimestampTimer;
- private final Timer commitTimer;
- private final Counter committedTxsCounter;
- private final Counter rolledbackTxsCounter;
- private final Counter errorTxsCounter;
- private final Counter invalidatedTxsCounter;
-
- /**
- * Base constructor
- *
- * @param metrics
- * instrumentation metrics
- * @param postCommitter
- * post commit action executor
- * @param tsoClient
- * a client for accessing functionality of the status oracle
- * @param commitTableClient
- * a client for accessing functionality of the commit table
- * @param transactionFactory
- * a transaction factory to create the specific transaction
- * objects required by the transaction manager being implemented.
- */
- public AbstractTransactionManager(MetricsRegistry metrics,
- PostCommitActions postCommitter,
- TSOClient tsoClient,
- CommitTable.Client commitTableClient,
- TransactionFactory<? extends CellId> transactionFactory) {
-
- this.tsoClient = tsoClient;
- this.postCommitter = postCommitter;
- this.commitTableClient = commitTableClient;
- this.transactionFactory = transactionFactory;
-
- // Metrics configuration
- this.startTimestampTimer = metrics.timer(name("omid", "tm", "hbase", "startTimestamp", "latency"));
- this.commitTimer = metrics.timer(name("omid", "tm", "hbase", "commit", "latency"));
- this.committedTxsCounter = metrics.counter(name("omid", "tm", "hbase", "committedTxs"));
- this.rolledbackTxsCounter = metrics.counter(name("omid", "tm", "hbase", "rolledbackTxs"));
- this.errorTxsCounter = metrics.counter(name("omid", "tm", "hbase", "erroredTxs"));
- this.invalidatedTxsCounter = metrics.counter(name("omid", "tm", "hbase", "invalidatedTxs"));
-
- }
-
- /**
- * Allows transaction manager developers to perform actions before creating a transaction.
- * @throws TransactionManagerException
- */
- public void preBegin() throws TransactionManagerException {}
-
- /**
- * @see com.yahoo.omid.transaction.TransactionManager#begin()
- */
- @Override
- public final Transaction begin() throws TransactionException {
-
- try {
- preBegin();
-
- long startTimestamp, epoch;
-
- // The loop is required for HA scenarios where we get the timestamp
- // but when getting the epoch, the client is connected to a new TSOServer
- // When this happen, the epoch will be larger than the startTimestamp,
- // so we need to start the transaction again. We use the fact that epoch
- // is always smaller or equal to a timestamp, and therefore, we first need
- // to get the timestamp and then the epoch.
- startTimestampTimer.start();
- try {
- do {
- startTimestamp = tsoClient.getNewStartTimestamp().get();
- epoch = tsoClient.getEpoch();
- } while (epoch > startTimestamp);
- } finally {
- startTimestampTimer.stop();
- }
-
- AbstractTransaction<? extends CellId> tx = transactionFactory.createTransaction(startTimestamp, epoch, this);
-
- postBegin(tx);
-
- return tx;
- } catch (TransactionManagerException e) {
- throw new TransactionException("An error has occured during PreBegin/PostBegin", e);
- } catch (ExecutionException e) {
- throw new TransactionException("Could not get new timestamp", e);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new TransactionException("Interrupted getting timestamp", ie);
- }
- }
-
- /**
- * Allows transaction manager developers to perform actions after having started a transaction.
- * @param transaction
- * the transaction that was just created.
- * @throws TransactionManagerException
- */
- public void postBegin(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
-
- /**
- * Allows transaction manager developers to perform actions before committing a transaction.
- * @param transaction
- * the transaction that is going to be committed.
- * @throws TransactionManagerException
- */
- public void preCommit(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
-
- /**
- * @see com.yahoo.omid.transaction.TransactionManager#commit(Transaction)
- */
- @Override
- public final void commit(Transaction transaction) throws RollbackException, TransactionException {
-
- AbstractTransaction<? extends CellId> tx = enforceAbstractTransactionAsParam(transaction);
- enforceTransactionIsInRunningState(tx);
-
- if (tx.isRollbackOnly()) { // Manage explicit user rollback
- rollback(tx);
- throw new RollbackException(tx + ": Tx was set to rollback explicitly");
- }
-
- try {
-
- preCommit(tx);
-
- commitTimer.start();
- try {
- if (tx.getWriteSet().isEmpty()) {
- markReadOnlyTransaction(tx); // No need for read-only transactions to contact the TSO Server
- } else {
- commitRegularTransaction(tx);
- }
- committedTxsCounter.inc();
- } finally {
- commitTimer.stop();
- }
-
- postCommit(tx);
-
- } catch (TransactionManagerException e) {
- throw new TransactionException(e.getMessage(), e);
- }
-
- }
-
- /**
- * Allows transaction manager developers to perform actions after committing a transaction.
- * @param transaction
- * the transaction that was committed.
- * @throws TransactionManagerException
- */
- public void postCommit(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
-
- /**
- * Allows transaction manager developers to perform actions before rolling-back a transaction.
- * @param transaction
- * the transaction that is going to be rolled-back.
- * @throws TransactionManagerException
- */
- public void preRollback(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
-
- /**
- * @see com.yahoo.omid.transaction.TransactionManager#rollback(Transaction)
- */
- @Override
- public final void rollback(Transaction transaction) throws TransactionException {
-
- AbstractTransaction<? extends CellId> tx = enforceAbstractTransactionAsParam(transaction);
- enforceTransactionIsInRunningState(tx);
-
- try {
-
- preRollback(tx);
-
- // Make sure its commit timestamp is 0, so the cleanup does the right job
- tx.setCommitTimestamp(0);
- tx.setStatus(Status.ROLLEDBACK);
-
- postRollback(tx);
-
- } catch (TransactionManagerException e) {
- throw new TransactionException(e.getMessage(), e);
- } finally {
- tx.cleanup();
- }
-
- }
-
- /**
- * Allows transaction manager developers to perform actions after rolling-back a transaction.
- * @param transaction
- * the transaction that was rolled-back.
- * @throws TransactionManagerException
- */
- public void postRollback(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
-
- /**
- * Check if the transaction commit data is in the shadow cell
- * @param cellStartTimestamp
- * the transaction start timestamp
- * locator
- * the timestamp locator
- * @throws IOException
- */
- Optional<CommitTimestamp> readCommitTimestampFromShadowCell(long cellStartTimestamp, CommitTimestampLocator locator)
- throws IOException
- {
-
- Optional<CommitTimestamp> commitTS = Optional.absent();
-
- Optional<Long> commitTimestamp = locator.readCommitTimestampFromShadowCell(cellStartTimestamp);
- if (commitTimestamp.isPresent()) {
- commitTS = Optional.of(new CommitTimestamp(SHADOW_CELL, commitTimestamp.get(), true)); // Valid commit TS
- }
-
- return commitTS;
- }
-
- /**
- * This function returns the commit timestamp for a particular cell if the transaction was already committed in
- * the system. In case the transaction was not committed and the cell was written by transaction initialized by a
- * previous TSO server, an invalidation try occurs.
- * Otherwise the function returns a value that indicates that the commit timestamp was not found.
- * @param cellStartTimestamp
- * start timestamp of the cell to locate the commit timestamp for.
- * @param epoch
- * the epoch of the TSO server the current tso client is working with.
- * @param locator
- * a locator to find the commit timestamp in the system.
- * @return the commit timestamp joint with the location where it was found
- * or an object indicating that it was not found in the system
- * @throws IOException
- */
- public CommitTimestamp locateCellCommitTimestamp(long cellStartTimestamp, long epoch,
- CommitTimestampLocator locator) throws IOException {
-
- try {
- // 1) First check the cache
- Optional<Long> commitTimestamp = locator.readCommitTimestampFromCache(cellStartTimestamp);
- if (commitTimestamp.isPresent()) { // Valid commit timestamp
- return new CommitTimestamp(CACHE, commitTimestamp.get(), true);
- }
-
- // 2) Then check the commit table
- // If the data was written at a previous epoch, check whether the transaction was invalidated
- Optional<CommitTimestamp> commitTimeStamp = commitTableClient.getCommitTimestamp(cellStartTimestamp).get();
- if (commitTimeStamp.isPresent()) {
- return commitTimeStamp.get();
- }
-
- // 3) Read from shadow cell
- commitTimeStamp = readCommitTimestampFromShadowCell(cellStartTimestamp, locator);
- if (commitTimeStamp.isPresent()) {
- return commitTimeStamp.get();
- }
-
- // 4) Check the epoch and invalidate the entry
- // if the data was written by a transaction from a previous epoch (previous TSO)
- if (cellStartTimestamp < epoch) {
- boolean invalidated = commitTableClient.tryInvalidateTransaction(cellStartTimestamp).get();
- if (invalidated) { // Invalid commit timestamp
- return new CommitTimestamp(COMMIT_TABLE, CommitTable.INVALID_TRANSACTION_MARKER, false);
- }
- }
-
- // 5) We did not manage to invalidate the transactions then check the commit table
- commitTimeStamp = commitTableClient.getCommitTimestamp(cellStartTimestamp).get();
- if (commitTimeStamp.isPresent()) {
- return commitTimeStamp.get();
- }
-
- // 6) Read from shadow cell
- commitTimeStamp = readCommitTimestampFromShadowCell(cellStartTimestamp, locator);
- if (commitTimeStamp.isPresent()) {
- return commitTimeStamp.get();
- }
-
- // *) Otherwise return not found
- return new CommitTimestamp(NOT_PRESENT, -1L /** TODO Check if we should return this */, true);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new IOException("Interrupted while finding commit timestamp", e);
- } catch (ExecutionException e) {
- throw new IOException("Problem finding commit timestamp", e);
- }
-
- }
-
- /**
- * @see java.io.Closeable#close()
- */
- @Override
- public final void close() throws IOException {
-
- tsoClient.close();
- commitTableClient.close();
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private void enforceTransactionIsInRunningState(Transaction transaction) {
-
- if (transaction.getStatus() != Status.RUNNING) {
- throw new IllegalArgumentException("Transaction was already " + transaction.getStatus());
- }
-
- }
-
- @SuppressWarnings("unchecked")
- // NOTE: We are sure that tx is not parametrized
- private AbstractTransaction<? extends CellId> enforceAbstractTransactionAsParam(Transaction tx) {
-
- if (tx instanceof AbstractTransaction) {
- return (AbstractTransaction<? extends CellId>) tx;
- } else {
- throw new IllegalArgumentException(
- "The transaction object passed is not an instance of AbstractTransaction");
- }
-
- }
-
- private void markReadOnlyTransaction(AbstractTransaction<? extends CellId> readOnlyTx) {
-
- readOnlyTx.setStatus(Status.COMMITTED_RO);
-
- }
-
- private void commitRegularTransaction(AbstractTransaction<? extends CellId> tx)
- throws RollbackException, TransactionException
- {
-
- try {
-
- long commitTs = tsoClient.commit(tx.getStartTimestamp(), tx.getWriteSet()).get();
- certifyCommitForTx(tx, commitTs);
- updateShadowCellsAndRemoveCommitTableEntry(tx, postCommitter);
-
- } catch (ExecutionException e) {
-
- if (e.getCause() instanceof AbortException) { // TSO reports Tx conflicts as AbortExceptions in the future
- rollback(tx);
- rolledbackTxsCounter.inc();
- throw new RollbackException("Conflicts detected in tx writeset", e.getCause());
- }
-
- if (e.getCause() instanceof ServiceUnavailableException
- ||
- e.getCause() instanceof NewTSOException
- ||
- e.getCause() instanceof ConnectionException) {
-
- errorTxsCounter.inc();
- try {
- LOG.warn("Can't contact the TSO for receiving outcome for Tx {}. Checking Commit Table...", tx);
- // Check the commit table to find if the target TSO woke up in the meantime and added the commit
- // TODO: Decide what we should we do if we can not contact the commit table
- Optional<CommitTimestamp> commitTimestamp =
- commitTableClient.getCommitTimestamp(tx.getStartTimestamp()).get();
- if (commitTimestamp.isPresent()) {
- if (commitTimestamp.get().isValid()) {
- LOG.warn("{}: Valid commit TS found in Commit Table. Committing Tx...", tx);
- certifyCommitForTx(tx, commitTimestamp.get().getValue());
- postCommitter.updateShadowCells(tx); // But do NOT remove transaction from commit table
- } else { // Probably another Tx in a new TSO Server invalidated this transaction
- LOG.warn("{}: Invalidated commit TS found in Commit Table. Rolling-back...", tx);
- rollback(tx);
- throw new RollbackException(tx + " invalidated by other Tx started", e.getCause());
- }
- } else {
- LOG.warn("{}: Trying to invalidate Tx proactively in Commit Table...", tx);
- boolean invalidated = commitTableClient.tryInvalidateTransaction(tx.getStartTimestamp()).get();
- if (invalidated) {
- LOG.warn("{}: Invalidated proactively in Commit Table. Rolling-back Tx...", tx);
- invalidatedTxsCounter.inc();
- rollback(tx); // Rollback proactively cause it's likely that a new TSOServer is now master
- throw new RollbackException(tx + " rolled-back precautionary", e.getCause());
- } else {
- LOG.warn("{}: Invalidation could NOT be completed. Re-checking Commit Table...", tx);
- // TODO: Decide what we should we do if we can not contact the commit table
- commitTimestamp = commitTableClient.getCommitTimestamp(tx.getStartTimestamp()).get();
- if (commitTimestamp.isPresent() && commitTimestamp.get().isValid()) {
- LOG.warn("{}: Valid commit TS found in Commit Table. Committing Tx...", tx);
- certifyCommitForTx(tx, commitTimestamp.get().getValue());
- postCommitter.updateShadowCells(tx); // But do NOT remove transaction from commit table
- } else {
- LOG.error("{}: Can't determine Transaction outcome", tx);
- throw new TransactionException(tx + ": cannot determine Tx outcome");
- }
- }
- }
- } catch (ExecutionException e1) {
- throw new TransactionException(tx + ": problem reading commitTS from Commit Table", e1);
- } catch (InterruptedException e1) {
- Thread.currentThread().interrupt();
- throw new TransactionException(tx + ": interrupted while reading commitTS from Commit Table", e1);
- }
- } else {
- throw new TransactionException(tx + ": cannot determine Tx outcome", e.getCause());
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new TransactionException(tx + ": interrupted during commit", ie);
-
- }
-
- }
-
- private void updateShadowCellsAndRemoveCommitTableEntry(final AbstractTransaction<? extends CellId> tx,
- final PostCommitActions postCommitter) {
-
- Futures.transform(postCommitter.updateShadowCells(tx), new Function<Void, Void>() {
- @Override
- public Void apply(Void aVoid) {
- postCommitter.removeCommitTableEntry(tx);
- return null;
- }
- });
-
- }
-
- private void certifyCommitForTx(AbstractTransaction<? extends CellId> txToSetup, long commitTS) {
-
- txToSetup.setStatus(Status.COMMITTED);
- txToSetup.setCommitTimestamp(commitTS);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
deleted file mode 100644
index 13399c5..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/CommitTimestampLocator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-
-import java.io.IOException;
-
-/**
- * An behavior that needs to be implemented by transaction managers
- * to try to locate the possible commit timestamp for a cell. It
- * allows to the transaction managers implementing this interface to
- * search the commit timestamp in a local cache of their own or in
- * their particular shadow cells implementation.
- */
-public interface CommitTimestampLocator {
-
- public Optional<Long> readCommitTimestampFromCache(long startTimestamp);
-
- public Optional<Long> readCommitTimestampFromShadowCell(long startTimestamp)
- throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
deleted file mode 100644
index 6b7b2ba..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/PostCommitActions.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.yahoo.omid.tso.client.CellId;
-
-public interface PostCommitActions {
-
- /**
- * Allows specific implementations to update the shadow cells.
- * @param transaction
- * the transaction to update shadow cells for
- * @return future signalling end of the computation
- */
- ListenableFuture<Void> updateShadowCells(AbstractTransaction<? extends CellId> transaction);
-
- /**
- * Allows specific implementations to remove the transaction entry from the commit table.
- * @param transaction
- * the transaction to remove the commit table entry
- * @return future signalling end of the computation
- */
- ListenableFuture<Void> removeCommitTableEntry(AbstractTransaction<? extends CellId> transaction);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
deleted file mode 100644
index 1cd1176..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/RollbackException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-public class RollbackException extends Exception {
-
- private static final long serialVersionUID = -9163407697376986830L;
-
- public RollbackException(String message) {
- super(message);
- }
-
- public RollbackException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public RollbackException() {
- super();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
deleted file mode 100644
index a10e2d3..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/Transaction.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-
-/**
- * This interface defines the transaction state & behavior exposed to users.
- */
-public interface Transaction {
-
- enum Status {
- RUNNING, COMMITTED, ROLLEDBACK, COMMITTED_RO
- }
-
- /**
- * Returns the transaction identifier
- * @return the transaction identifier
- */
- long getTransactionId();
-
- /**
- * Returns the epoch given by the TSOServer
- * @return the transaction's TSOServer epoch
- */
- long getEpoch();
-
- /**
- * Returns the current transaction {@link Status}
- * @return transaction status
- */
- Status getStatus();
-
- /**
- * Forces the transaction to rollback, even when there's an intention
- * to commit it.
- */
- void setRollbackOnly();
-
- /**
- * Returns whether the transaction was marked for rollback or not
- * @return whether the transaction is marked for rollback or not
- */
- boolean isRollbackOnly();
-
-
- /**
- * Set of methods to attach some metadata to a transaction object. One example
- * of such metadata are notifications
- *
- *
- * Expects they metadata stored under key "key" to be of the "Set" type,
- * append "value" to the existing set or creates a new one
- */
- void appendMetadata(String key, Object value);
-
- void setMetadata(String key, Object value);
-
- Optional<Object> getMetadata(String key);
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
deleted file mode 100644
index f238a6b..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionException.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-public class TransactionException extends Exception {
-
- private static final long serialVersionUID = 7273525983622126275L;
-
- public TransactionException(String reason) {
- super(reason);
- }
-
- public TransactionException(String reason, Throwable e) {
- super(reason, e);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
deleted file mode 100644
index d5782fa..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import java.io.Closeable;
-
-/**
- * Provides the methods to manage transactions (create, commit...)
- */
-public interface TransactionManager extends Closeable {
-
- /**
- * Starts a new transaction.
- *
- * Creates & returns a {@link Transaction} interface implementation
- * that will be used in {@link TTable}'s methods for doing operations
- * on the transactional context defined by the returned object.
- *
- * @return transaction
- * representation of the created transaction
- * @throws TransactionException
- */
- public Transaction begin() throws TransactionException;
-
- /**
- * Commits a transaction.
- *
- * If the transaction was marked for rollback or has conflicts
- * with another concurrent transaction it will be rolledback
- * automatically and a {@link RollbackException} will be thrown.
- *
- * @param tx
- * transaction to be committed.
- * @throws RollbackException
- * thrown when transaction has conflicts with another transaction
- * or when was marked for rollback.
- * @throws TransactionException
- */
- public void commit(Transaction tx) throws RollbackException, TransactionException;
-
- /**
- * Aborts a transaction.
- *
- * Automatically rollbacks the changes performed by the transaction.
- *
- * @param tx
- * transaction to be rolled-back.
- */
- public void rollback(Transaction tx) throws TransactionException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java b/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
deleted file mode 100644
index 09df8da..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/transaction/TransactionManagerException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-public class TransactionManagerException extends Exception {
-
- private static final long serialVersionUID = 6811947817962260774L;
-
- public TransactionManagerException(String reason) {
- super(reason);
- }
-
- public TransactionManagerException(String reason, Throwable e) {
- super(reason, e);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java
deleted file mode 100644
index 670e87f..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/client/AbortException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tso.client;
-
-/**
- * Thrown when the TSO server has aborted a transaction
- */
-public class AbortException extends Exception {
-
- private static final long serialVersionUID = 1861474360100681040L;
-
-}
\ No newline at end of file
[04/52] [abbrv] incubator-omid git commit: Updated source code with
license header
Posted by ik...@apache.org.
Updated source code with license header
Change-Id: Ib1b5c1ad6cdb27ffaf7feb6184ff48bc1aab024c
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/0f13d2ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/0f13d2ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/0f13d2ad
Branch: refs/heads/master
Commit: 0f13d2ad3d83e2bfce6ad22d8bbbad685a05e13c
Parents: a5b4f42
Author: Igor Katkov <ka...@yahoo-inc.com>
Authored: Tue Apr 12 16:17:24 2016 -0700
Committer: Igor Katkov <ka...@yahoo-inc.com>
Committed: Tue Apr 12 16:17:24 2016 -0700
----------------------------------------------------------------------
.../benchmarks/hbase/HBaseCommitTableTester.java | 14 ++++++++------
.../yahoo/omid/benchmarks/tso/RawTxRunner.java | 14 ++++++++------
.../omid/benchmarks/tso/TSOServerBenchmark.java | 14 ++++++++------
.../benchmarks/tso/TSOServerBenchmarkConfig.java | 14 ++++++++------
.../omid/benchmarks/utils/UniformGenerator.java | 14 ++++++++------
.../omid/metrics/CodahaleMetricsConfig.java | 14 ++++++++------
.../omid/metrics/CodahaleMetricsProvider.java | 14 ++++++++------
.../com/yahoo/omid/committable/CommitTable.java | 14 ++++++++------
.../omid/committable/InMemoryCommitTable.java | 14 ++++++++------
.../yahoo/omid/committable/NullCommitTable.java | 14 ++++++++------
.../omid/committable/NullCommitTableTest.java | 17 +++++++++++++++++
.../src/main/java/com/yahoo/omid/YAMLUtils.java | 14 ++++++++------
.../src/main/java/com/yahoo/omid/zk/ZKUtils.java | 14 ++++++++------
common/src/main/proto/TSOProto.proto | 14 ++++++++------
.../test/java/com/yahoo/omid/YAMLUtilsTest.java | 17 +++++++++++++++++
.../com/yahoo/omid/examples/BasicExample.java | 14 ++++++++------
.../omid/examples/ConfigurationExample.java | 14 ++++++++------
.../omid/examples/SnapshotIsolationExample.java | 14 ++++++++------
.../yahoo/omid/transaction/ColumnWrapper.java | 14 ++++++++------
.../transaction/HBaseAsyncPostCommitter.java | 14 ++++++++------
.../com/yahoo/omid/transaction/HBaseCellId.java | 14 ++++++++------
.../HBaseOmidClientConfiguration.java | 14 ++++++++------
.../omid/transaction/HBaseSyncPostCommitter.java | 14 ++++++++------
.../yahoo/omid/transaction/HBaseTransaction.java | 14 ++++++++------
.../omid/transaction/HBaseTransactionClient.java | 14 ++++++++------
.../transaction/HBaseTransactionManager.java | 14 ++++++++------
.../java/com/yahoo/omid/transaction/TTable.java | 14 ++++++++------
.../transaction/TTableCellGetterAdapter.java | 14 ++++++++------
.../com/yahoo/omid/transaction/OmidTestBase.java | 17 +++++++++++++++++
.../TestAsynchronousPostCommitter.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestAutoFlush.java | 17 +++++++++++++++++
.../omid/transaction/TestBasicTransaction.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestCellUtils.java | 17 +++++++++++++++++
.../omid/transaction/TestColumnIterator.java | 17 +++++++++++++++++
.../com/yahoo/omid/transaction/TestDeletion.java | 17 +++++++++++++++++
.../transaction/TestEndToEndScenariosWithHA.java | 17 +++++++++++++++++
.../com/yahoo/omid/transaction/TestFilters.java | 17 +++++++++++++++++
.../transaction/TestHALeaseManagementModule.java | 14 ++++++++------
.../TestHBaseOmidClientConfiguration.java | 17 +++++++++++++++++
.../transaction/TestHBaseTransactionClient.java | 17 +++++++++++++++++
.../transaction/TestHBaseTransactionManager.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestMultiplePut.java | 17 +++++++++++++++++
.../com/yahoo/omid/transaction/TestReadPath.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestShadowCells.java | 17 +++++++++++++++++
.../omid/transaction/TestSingleColumnFamily.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestTSOModule.java | 17 +++++++++++++++++
.../omid/transaction/TestTTableBehaviour.java | 17 +++++++++++++++++
.../omid/transaction/TestTransactionCleanup.java | 17 +++++++++++++++++
.../transaction/TestTransactionConflict.java | 17 +++++++++++++++++
.../omid/transaction/TestTxMgrFailover.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestUpdateScan.java | 17 +++++++++++++++++
.../DefaultHBaseCommitTableStorageModule.java | 14 ++++++++------
.../omid/committable/hbase/HBaseCommitTable.java | 14 ++++++++------
.../hbase/HBaseCommitTableConfig.java | 14 ++++++++------
.../hbase/HBaseCommitTableStorageModule.java | 14 ++++++++------
.../committable/hbase/TestHBaseCommitTable.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/HBaseConfigModule.java | 14 ++++++++------
.../omid/committable/hbase/KeyGenerator.java | 14 ++++++++------
.../hbase/KeyGeneratorImplementations.java | 14 ++++++++------
.../com/yahoo/omid/tools/hbase/HBaseLogin.java | 14 ++++++++------
.../omid/tools/hbase/SecureHBaseConfig.java | 14 ++++++++------
.../com/yahoo/omid/transaction/CellInfo.java | 14 ++++++++------
.../com/yahoo/omid/transaction/CellUtils.java | 14 ++++++++------
.../hbase/TestHBaseCommitTableKey.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/CompactorUtil.java | 14 ++++++++------
.../transaction/HRegionCellGetterAdapter.java | 14 ++++++++------
.../yahoo/omid/transaction/OmidCompactor.java | 14 ++++++++------
.../hbase/regionserver/CompactorScanner.java | 14 ++++++++------
.../TSOForHBaseCompactorTestModule.java | 17 +++++++++++++++++
.../yahoo/omid/transaction/TestCompaction.java | 17 +++++++++++++++++
.../omid/transaction/TestCompactorScanner.java | 17 +++++++++++++++++
hbase-coprocessor/src/test/resources/log4j.xml | 19 +++++++++++++++++++
.../src/main/java/com/yahoo/omid/HBaseShims.java | 14 ++++++++------
.../apache/hadoop/hbase/regionserver/Region.java | 14 ++++++++------
.../hbase/regionserver/ScannerContext.java | 14 ++++++++------
.../src/main/java/com/yahoo/omid/HBaseShims.java | 14 ++++++++------
.../yahoo/omid/tools/hbase/OmidTableManager.java | 14 ++++++++------
.../omid/tools/hbase/TestOmidTableManager.java | 17 +++++++++++++++++
.../omid/metrics/AbstractMetricsConfig.java | 14 ++++++++------
.../java/com/yahoo/omid/metrics/Counter.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Gauge.java | 14 ++++++++------
.../java/com/yahoo/omid/metrics/Histogram.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Meter.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Metric.java | 14 ++++++++------
.../com/yahoo/omid/metrics/MetricsProvider.java | 14 ++++++++------
.../com/yahoo/omid/metrics/MetricsRegistry.java | 14 ++++++++------
.../yahoo/omid/metrics/MetricsRegistryMap.java | 14 ++++++++------
.../com/yahoo/omid/metrics/MetricsUtils.java | 14 ++++++++------
.../yahoo/omid/metrics/NullMetricsProvider.java | 14 ++++++++------
.../main/java/com/yahoo/omid/metrics/Timer.java | 14 ++++++++------
misc/header.txt | 14 ++++++++------
pom.xml | 12 +-----------
.../com/yahoo/statemachine/StateMachine.java | 14 ++++++++------
.../statemachine/StateMachineLogParser.java | 14 ++++++++------
.../com/yahoo/statemachine/TestStateMachine.java | 17 +++++++++++++++++
.../DefaultHBaseTimestampStorageModule.java | 14 ++++++++------
.../storage/DefaultZKTimestampStorageModule.java | 14 ++++++++------
.../timestamp/storage/HBaseTimestampStorage.java | 14 ++++++++------
.../storage/HBaseTimestampStorageConfig.java | 14 ++++++++------
.../storage/HBaseTimestampStorageModule.java | 14 ++++++++------
.../omid/timestamp/storage/TimestampStorage.java | 14 ++++++++------
.../yahoo/omid/timestamp/storage/ZKModule.java | 14 ++++++++------
.../omid/timestamp/storage/ZKTimestampPaths.java | 14 ++++++++------
.../timestamp/storage/ZKTimestampStorage.java | 14 ++++++++------
.../storage/ZKTimestampStorageModule.java | 14 ++++++++------
.../storage/TestHBaseTimestampStorage.java | 17 +++++++++++++++++
.../storage/TestZKTimestampStorage.java | 17 +++++++++++++++++
.../omid/transaction/AbstractTransaction.java | 14 ++++++++------
.../transaction/AbstractTransactionManager.java | 14 ++++++++------
.../omid/transaction/CommitTimestampLocator.java | 14 ++++++++------
.../omid/transaction/PostCommitActions.java | 14 ++++++++------
.../omid/transaction/RollbackException.java | 14 ++++++++------
.../com/yahoo/omid/transaction/Transaction.java | 14 ++++++++------
.../omid/transaction/TransactionException.java | 14 ++++++++------
.../omid/transaction/TransactionManager.java | 14 ++++++++------
.../transaction/TransactionManagerException.java | 14 ++++++++------
.../com/yahoo/omid/tso/util/DummyCellIdImpl.java | 14 ++++++++------
.../com/yahoo/omid/tsoclient/AbortException.java | 14 ++++++++------
.../java/com/yahoo/omid/tsoclient/CellId.java | 14 ++++++++------
.../yahoo/omid/tsoclient/ClosingException.java | 14 ++++++++------
.../omid/tsoclient/ConnectionException.java | 14 ++++++++------
.../omid/tsoclient/ForwardingTSOFuture.java | 14 ++++++++------
.../omid/tsoclient/HandshakeFailedException.java | 14 ++++++++------
.../com/yahoo/omid/tsoclient/MockTSOClient.java | 14 ++++++++------
.../yahoo/omid/tsoclient/NewTSOException.java | 14 ++++++++------
.../omid/tsoclient/OmidClientConfiguration.java | 14 ++++++++------
.../tsoclient/ServiceUnavailableException.java | 14 ++++++++------
.../java/com/yahoo/omid/tsoclient/TSOClient.java | 14 ++++++++------
.../java/com/yahoo/omid/tsoclient/TSOFuture.java | 14 ++++++++------
.../com/yahoo/omid/tsoclient/TSOProtocol.java | 14 ++++++++------
.../yahoo/omid/tsoclient/TestMockTSOClient.java | 17 +++++++++++++++++
.../tsoclient/TestOmidClientConfiguration.java | 17 +++++++++++++++++
.../src/main/java/com/yahoo/omid/tso/Cache.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/CacheEvaluation.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/CommitHashMap.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/DisruptorModule.java | 14 ++++++++------
.../yahoo/omid/tso/FatalExceptionHandler.java | 14 ++++++++------
.../yahoo/omid/tso/HALeaseManagementModule.java | 14 ++++++++------
.../tso/InMemoryCommitTableStorageModule.java | 14 ++++++++------
.../omid/tso/InMemoryTimestampStorageModule.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/LeaseManagement.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/LeaseManager.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/LongCache.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/MockPanicker.java | 14 ++++++++------
.../com/yahoo/omid/tso/MonitoringContext.java | 14 ++++++++------
.../yahoo/omid/tso/NetworkInterfaceUtils.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/Panicker.java | 14 ++++++++------
.../yahoo/omid/tso/PausableTimestampOracle.java | 14 ++++++++------
.../com/yahoo/omid/tso/PersistenceProcessor.java | 14 ++++++++------
.../yahoo/omid/tso/PersistenceProcessorImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/ReplyProcessor.java | 14 ++++++++------
.../com/yahoo/omid/tso/ReplyProcessorImpl.java | 14 ++++++++------
.../com/yahoo/omid/tso/RequestProcessor.java | 14 ++++++++------
.../com/yahoo/omid/tso/RequestProcessorImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/RetryProcessor.java | 14 ++++++++------
.../com/yahoo/omid/tso/RetryProcessorImpl.java | 14 ++++++++------
.../com/yahoo/omid/tso/SystemExitPanicker.java | 14 ++++++++------
.../com/yahoo/omid/tso/TSOChannelHandler.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/TSOModule.java | 14 ++++++++------
.../main/java/com/yahoo/omid/tso/TSOServer.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TSOServerConfig.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TSOStateManager.java | 14 ++++++++------
.../com/yahoo/omid/tso/TSOStateManagerImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TimestampOracle.java | 14 ++++++++------
.../com/yahoo/omid/tso/TimestampOracleImpl.java | 14 ++++++++------
.../java/com/yahoo/omid/tso/TsoServerDaemon.java | 14 ++++++++------
.../omid/tso/VoidLeaseManagementModule.java | 14 ++++++++------
.../com/yahoo/omid/tso/VoidLeaseManager.java | 14 ++++++++------
.../src/test/java/com/yahoo/omid/TestUtils.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/PausableLeaseManager.java | 17 +++++++++++++++++
.../yahoo/omid/tso/ProgrammableTSOServer.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/tso/TSOMockModule.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TSOServerConfigTest.java | 17 +++++++++++++++++
.../test/java/com/yahoo/omid/tso/TestBatch.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestLeaseManager.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/tso/TestLongCache.java | 17 +++++++++++++++++
.../java/com/yahoo/omid/tso/TestPanicker.java | 17 +++++++++++++++++
.../yahoo/omid/tso/TestPersistenceProcessor.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestRequestProcessor.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestRetryProcessor.java | 17 +++++++++++++++++
.../omid/tso/TestTSOChannelHandlerNetty.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestTSOStateManager.java | 17 +++++++++++++++++
.../com/yahoo/omid/tso/TestTimestampOracle.java | 17 +++++++++++++++++
.../yahoo/omid/tsoclient/TSOClientAccessor.java | 17 +++++++++++++++++
.../yahoo/omid/tsoclient/TSOClientOneShot.java | 17 +++++++++++++++++
.../com/yahoo/omid/tsoclient/TSOClientRaw.java | 17 +++++++++++++++++
...ationOfTSOClientServerBasicFunctionality.java | 17 +++++++++++++++++
.../tsoclient/TestTSOClientConnectionToTSO.java | 17 +++++++++++++++++
...estTSOClientRequestAndResponseBehaviours.java | 17 +++++++++++++++++
.../tsoclient/TestTSOClientResponseHandling.java | 17 +++++++++++++++++
.../omid/tsoclient/TestUnconnectedTSOClient.java | 17 +++++++++++++++++
191 files changed, 2054 insertions(+), 797 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
index 9e7bc26..cd690e2 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/hbase/HBaseCommitTableTester.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
index de0fbe7..4082b61 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/RawTxRunner.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
index 94f212b..b39d28a 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmark.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
index 2fa02a0..ba42544 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/tso/TSOServerBenchmarkConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
----------------------------------------------------------------------
diff --git a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
index d9b56c2..eb27bd3 100644
--- a/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
+++ b/benchmarks/src/main/java/com/yahoo/omid/benchmarks/utils/UniformGenerator.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
----------------------------------------------------------------------
diff --git a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java b/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
index 82f7c39..d251b66 100644
--- a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
+++ b/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsConfig.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
----------------------------------------------------------------------
diff --git a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java b/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
index 6535880..751ce34 100644
--- a/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
+++ b/codahale-metrics/src/main/java/com/yahoo/omid/metrics/CodahaleMetricsProvider.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java b/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
index 4bfcd4f..78ae44b 100644
--- a/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
+++ b/commit-table/src/main/java/com/yahoo/omid/committable/CommitTable.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java b/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
index 7ba8f57..181e7aa 100644
--- a/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
+++ b/commit-table/src/main/java/com/yahoo/omid/committable/InMemoryCommitTable.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
----------------------------------------------------------------------
diff --git a/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java b/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
index 2ab5583..26aac54 100644
--- a/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
+++ b/commit-table/src/main/java/com/yahoo/omid/committable/NullCommitTable.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
----------------------------------------------------------------------
diff --git a/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java b/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
index 7e56fe9..92cd248 100644
--- a/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
+++ b/commit-table/src/test/java/com/yahoo/omid/committable/NullCommitTableTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.committable;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/common/src/main/java/com/yahoo/omid/YAMLUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/yahoo/omid/YAMLUtils.java b/common/src/main/java/com/yahoo/omid/YAMLUtils.java
index 994d583..6edd71d 100644
--- a/common/src/main/java/com/yahoo/omid/YAMLUtils.java
+++ b/common/src/main/java/com/yahoo/omid/YAMLUtils.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java b/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
index 240ec98..10e3df1 100644
--- a/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
+++ b/common/src/main/java/com/yahoo/omid/zk/ZKUtils.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/common/src/main/proto/TSOProto.proto
----------------------------------------------------------------------
diff --git a/common/src/main/proto/TSOProto.proto b/common/src/main/proto/TSOProto.proto
index e26a863..23762c5 100644
--- a/common/src/main/proto/TSOProto.proto
+++ b/common/src/main/proto/TSOProto.proto
@@ -1,11 +1,13 @@
//
-// Copyright 2011-2016 Yahoo Inc.
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java b/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
index 3758b85..172e898 100644
--- a/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
+++ b/common/src/test/java/com/yahoo/omid/YAMLUtilsTest.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java b/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
index 2e8c68c..44d9646 100644
--- a/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
+++ b/examples/src/main/java/com/yahoo/omid/examples/BasicExample.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java b/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
index 4ec2785..c2dc663 100644
--- a/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
+++ b/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java b/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
index 1cffe2b..81ed552 100644
--- a/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
+++ b/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
index 1d1698c..2e6a94f 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
index 5b82935..091cf58 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
index 1922f6d..c398fb4 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
index 76ca33b..fbc8119 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
index be20c34..0a90d68 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
index e3e708d..4acda7d 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
index e6de123..dd33832 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
index f87376f..fe21cd1 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
index c4935d0..53b62eb 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/TTable.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
index a60c4d1..77548a5 100644
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
+++ b/hbase-client/src/main/java/com/yahoo/omid/transaction/TTableCellGetterAdapter.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
index 7bf0c7c..61bf567 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.inject.Guice;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
index 7386442..ad3fad1 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
index dcced9b..c40d56d 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.client.Get;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
index a292151..65896cc 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.client.Get;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
index 5591760..3e39d69 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestCellUtils.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
index eb0e61e..e57c7fe 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestColumnIterator.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.collect.ImmutableList;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
index 6cd2c08..c3c24bb 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestDeletion.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.client.Delete;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
index 825c5a0..8c060fb 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestEndToEndScenariosWithHA.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Charsets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
index c71ee6e..7a3a636 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestFilters.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.ListenableFuture;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
index bc5b342..195f4aa 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHALeaseManagementModule.java
@@ -1,11 +1,13 @@
/**
- * Copyright 2011-2016 Yahoo Inc.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
index 78f1f0f..28bfe1e 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseOmidClientConfiguration.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
index 5d0ebac..9f3ce1c 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionClient.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
index 866a835..beb999d 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.yahoo.omid.tsoclient.TSOClient;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
index c9ca5aa..726a237 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.client.Get;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
index 7e737c1..9d36b7b 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.Cell;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
index d94365e..8d85dc5 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Charsets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
index 6cb4413..b8b28fc 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.client.Put;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
index 5f9e7de..dced462 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.inject.AbstractModule;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
index 52484d2..023ebc6 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Charsets;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
index f782975..863480e 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.util.concurrent.SettableFuture;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
index d6d1e11..627fe79 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import org.apache.hadoop.hbase.HColumnDescriptor;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/0f13d2ad/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
index be2d21f..bfa6b4d 100644
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
+++ b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.yahoo.omid.transaction;
import com.google.common.base.Optional;
[14/52] [abbrv] incubator-omid git commit: [ci skip]prepare for next
development iteration
Posted by ik...@apache.org.
[ci skip]prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/5734cac4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/5734cac4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/5734cac4
Branch: refs/heads/master
Commit: 5734cac4713076019a3cf503bdb6e507386d10e8
Parents: 6dd07af
Author: Omid CI <om...@yahoo-inc.com>
Authored: Fri Apr 15 19:09:34 2016 +0000
Committer: Omid CI <om...@yahoo-inc.com>
Committed: Fri Apr 15 19:09:34 2016 +0000
----------------------------------------------------------------------
benchmarks/pom.xml | 2 +-
codahale-metrics/pom.xml | 2 +-
commit-table/pom.xml | 2 +-
common/pom.xml | 2 +-
examples/pom.xml | 2 +-
hbase-client/pom.xml | 2 +-
hbase-commit-table/pom.xml | 2 +-
hbase-common/pom.xml | 2 +-
hbase-coprocessor/pom.xml | 2 +-
hbase-shims/hbase-0/pom.xml | 2 +-
hbase-shims/hbase-1/pom.xml | 2 +-
hbase-shims/pom.xml | 2 +-
hbase-tools/pom.xml | 2 +-
metrics/pom.xml | 2 +-
pom.xml | 4 ++--
statemachine/pom.xml | 2 +-
timestamp-storage/pom.xml | 2 +-
transaction-client/pom.xml | 2 +-
tso-server/pom.xml | 2 +-
19 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/benchmarks/pom.xml
----------------------------------------------------------------------
diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml
index c5f6c01..9991a3d 100644
--- a/benchmarks/pom.xml
+++ b/benchmarks/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>benchmarks</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/codahale-metrics/pom.xml
----------------------------------------------------------------------
diff --git a/codahale-metrics/pom.xml b/codahale-metrics/pom.xml
index 98656d6..bf9f3d9 100644
--- a/codahale-metrics/pom.xml
+++ b/codahale-metrics/pom.xml
@@ -4,7 +4,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/commit-table/pom.xml b/commit-table/pom.xml
index 9dd06a0..4ad3c4d 100644
--- a/commit-table/pom.xml
+++ b/commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 8e6d498..5a84a35 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 737f89a..592fe28 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index 28f5268..cab62e8 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-commit-table/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-commit-table/pom.xml b/hbase-commit-table/pom.xml
index 8ebac20..0ddfafd 100644
--- a/hbase-commit-table/pom.xml
+++ b/hbase-commit-table/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase-commit-table</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-common/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-common/pom.xml b/hbase-common/pom.xml
index 380a144..7be81f5 100644
--- a/hbase-common/pom.xml
+++ b/hbase-common/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase-common</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-coprocessor/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/pom.xml b/hbase-coprocessor/pom.xml
index e7b2948..8ac01a6 100644
--- a/hbase-coprocessor/pom.xml
+++ b/hbase-coprocessor/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase-coprocessor</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index e4449df..401b2e4 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase0-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index 6f58a75..f3de6f4 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase1-shims</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index 98eecab..d04bab3 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>omid-shims-aggregator</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index 9fa8235..3010241 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>hbase-tools</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index 1b097a9..7950f57 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -6,7 +6,7 @@
<parent>
<artifactId>omid</artifactId>
<groupId>com.yahoo.omid</groupId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>metrics</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b93e769..060d35f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,7 +13,7 @@
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
<licenses>
<license>
@@ -46,7 +46,7 @@
<connection>scm:git:git://github.com/yahoo/omid.git</connection>
<developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
<url>https://github.com/yahoo/omid</url>
- <tag>omid-0.8.1.36</tag>
+ <tag>omid-0.8.1.21</tag>
</scm>
<distributionManagement>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index 22f76b3..eff5fb2 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<name>State Machine</name>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/timestamp-storage/pom.xml
----------------------------------------------------------------------
diff --git a/timestamp-storage/pom.xml b/timestamp-storage/pom.xml
index c413f43..3e8e60e 100644
--- a/timestamp-storage/pom.xml
+++ b/timestamp-storage/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>timestamp-storage</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/transaction-client/pom.xml
----------------------------------------------------------------------
diff --git a/transaction-client/pom.xml b/transaction-client/pom.xml
index 7e4e8ff..f7c27d9 100644
--- a/transaction-client/pom.xml
+++ b/transaction-client/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>transaction-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/5734cac4/tso-server/pom.xml
----------------------------------------------------------------------
diff --git a/tso-server/pom.xml b/tso-server/pom.xml
index c06345a..2950b98 100644
--- a/tso-server/pom.xml
+++ b/tso-server/pom.xml
@@ -5,7 +5,7 @@
<parent>
<groupId>com.yahoo.omid</groupId>
<artifactId>omid</artifactId>
- <version>0.8.1.36</version>
+ <version>0.8.1.37-SNAPSHOT</version>
</parent>
<artifactId>tso-server</artifactId>
[37/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
deleted file mode 100644
index 052f7da..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestHBaseTransactionManager.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.yahoo.omid.tso.client.TSOClient;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anySetOf;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-// TODO These tests should be adapted to a future test for AbstractTransactionManager as they should be DB independent
-@Test(groups = "sharedHBase")
-public class TestHBaseTransactionManager extends OmidTestBase {
-
- private static final int FAKE_EPOCH_INCREMENT = 100;
-
- private final byte[] row1 = Bytes.toBytes(TestHBaseTransactionManager.class.getCanonicalName());
- private final byte[] testFamily = Bytes.toBytes(TEST_FAMILY);
- private final byte[] qualifier = Bytes.toBytes("TEST_Q");
- private final byte[] data1 = Bytes.toBytes("test_data1");
-
-
- @Test(timeOut = 20_000)
- public void testTxManagerGetsTimestampsInTheRightEpoch(ITestContext context) throws Exception {
-
- TSOClient tsoClient = spy(getClient(context));
-
- long fakeEpoch = tsoClient.getNewStartTimestamp().get() + FAKE_EPOCH_INCREMENT;
-
- // Modify the epoch before testing the begin method
- doReturn(fakeEpoch).when(tsoClient).getEpoch();
-
- AbstractTransactionManager tm = spy((AbstractTransactionManager) newTransactionManager(context, tsoClient));
-
- // Create a transaction with the initial setup and check that the TX id matches the fake epoch created
- Transaction tx1 = tm.begin();
- assertEquals(tx1.getTransactionId(), fakeEpoch);
- verify(tsoClient, timeout(100).times(FAKE_EPOCH_INCREMENT)).getEpoch();
-
- }
-
- @Test(timeOut = 20_000)
- public void testReadOnlyTransactionsDoNotContactTSOServer(ITestContext context) throws Exception {
-
- final int EXPECTED_INVOCATIONS_FOR_COMMIT = 1; // Test specific checks
-
- TSOClient tsoClient = spy(getClient(context));
- TransactionManager tm = newTransactionManager(context, tsoClient);
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Add initial data in a transactional context
- Transaction tx1 = tm.begin();
- Put put = new Put(row1);
- put.add(testFamily, qualifier, data1);
- txTable.put(tx1, put);
- tm.commit(tx1);
-
- verify(tsoClient, times(EXPECTED_INVOCATIONS_FOR_COMMIT)).commit(anyLong(), anySetOf(HBaseCellId.class));
-
- // Create a read-only tx and verify that commit has not been invoked again in the TSOClient
- AbstractTransaction readOnlyTx = (AbstractTransaction) tm.begin();
- Get get = new Get(row1);
- Result r = txTable.get(readOnlyTx, get);
- assertTrue(Bytes.equals(r.getValue(testFamily, qualifier), data1), "Wrong value for RO-TX " + readOnlyTx);
- assertTrue(readOnlyTx.getWriteSet().isEmpty());
- tm.commit(readOnlyTx);
-
- verify(tsoClient, times(EXPECTED_INVOCATIONS_FOR_COMMIT)).commit(anyLong(), anySetOf(HBaseCellId.class));
- assertEquals(readOnlyTx.getStatus(), Transaction.Status.COMMITTED_RO);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
deleted file mode 100644
index 726a237..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestMultiplePut.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestMultiplePut extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestMultiplePut.class);
-
- private static final byte[] testTable = Bytes.toBytes(TEST_TABLE);
- private static final byte[] family = Bytes.toBytes(TEST_FAMILY);
- private static final byte[] col1 = Bytes.toBytes("col1");
- private static final byte[] col2 = Bytes.toBytes("col2");
- private static final byte[] data = Bytes.toBytes("testData");
-
- @Test(timeOut = 30_000)
- public void testMultiPutInTwoDifferentColsOfSameRowAreInTheTableAfterCommit(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- Transaction tx = tm.begin();
-
- byte[] rowToAdd = Bytes.toBytes(1000);
-
- Put put1 = new Put(rowToAdd);
- put1.add(family, col1, data);
- txTable.put(tx, put1);
-
- Put put2 = new Put(rowToAdd);
- put2.add(family, col2, data);
- txTable.put(tx, put2);
-
- tm.commit(tx);
-
- assertTrue(verifyValue(testTable, rowToAdd, family, col1, data), "Invalid value in table");
- assertTrue(verifyValue(testTable, rowToAdd, family, col2, data), "Invalid value in table");
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testManyManyPutsInDifferentRowsAreInTheTableAfterCommit(ITestContext context) throws Exception {
-
- final int NUM_ROWS_TO_ADD = 50;
-
- TransactionManager tm = newTransactionManager(context);
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- Transaction tx = tm.begin();
-
- for (int i = 0; i <= NUM_ROWS_TO_ADD; i++) {
- byte[] rowToAdd = Bytes.toBytes(i);
- byte[] dataForRowCol = Bytes.toBytes("testData" + i);
- Put put = new Put(rowToAdd);
- put.add(family, col1, dataForRowCol);
- txTable.put(tx, put);
- }
-
- tm.commit(tx);
-
- // Check some of the added values are there in the table
- byte[] rowToCheck = Bytes.toBytes(0);
- byte[] dataToCheck = Bytes.toBytes("testData" + 0);
- assertTrue(verifyValue(testTable, rowToCheck, family, col1, dataToCheck), "Invalid value in table");
- rowToCheck = Bytes.toBytes(NUM_ROWS_TO_ADD / 2);
- dataToCheck = Bytes.toBytes("testData" + (NUM_ROWS_TO_ADD / 2));
- assertTrue(verifyValue(testTable, rowToCheck, family, col1, dataToCheck), "Invalid value in table");
- rowToCheck = Bytes.toBytes(NUM_ROWS_TO_ADD);
- dataToCheck = Bytes.toBytes("testData" + NUM_ROWS_TO_ADD);
- assertTrue(verifyValue(testTable, rowToCheck, family, col1, dataToCheck), "Invalid value in table");
-
- }
- }
-
- @Test(timeOut = 30_000)
- public void testGetFromNonExistentRowAfterMultiplePutsReturnsNoResult(ITestContext context) throws Exception {
-
- final int NUM_ROWS_TO_ADD = 10;
-
- TransactionManager tm = newTransactionManager(context);
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- Transaction tx = tm.begin();
-
- for (int i = 0; i < NUM_ROWS_TO_ADD; i++) {
- byte[] rowToAdd = Bytes.toBytes(i);
- Put put = new Put(rowToAdd);
- put.add(family, col1, Bytes.toBytes("testData" + i));
- txTable.put(tx, put);
- }
-
- byte[] nonExistentRow = Bytes.toBytes(NUM_ROWS_TO_ADD + 5);
- Get get = new Get(nonExistentRow);
- Result result = txTable.get(tx, get);
-
- assertTrue(result.isEmpty(), "Found a row that should not exist");
-
- tm.commit(tx);
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
deleted file mode 100644
index 9d36b7b..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestReadPath.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.testng.AssertJUnit.assertFalse;
-import static org.testng.AssertJUnit.assertNotNull;
-import static org.testng.AssertJUnit.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestReadPath extends OmidTestBase {
-
- final byte[] family = Bytes.toBytes(TEST_FAMILY);
- final byte[] row = Bytes.toBytes("row");
- private final byte[] col = Bytes.toBytes("col1");
- final byte[] data = Bytes.toBytes("data");
- private final byte[] uncommitted = Bytes.toBytes("uncommitted");
-
- @Test
- public void testReadInterleaved(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- // Put some data on the DB
- Transaction t1 = tm.begin();
- Transaction t2 = tm.begin();
-
- Put put = new Put(row);
- put.add(family, col, data);
- table.put(t1, put);
- tm.commit(t1);
-
- Get get = new Get(row);
- Result result = table.get(t2, get);
- assertFalse("Should be unable to read column", result.containsColumn(family, col));
- }
-
- @Test
- public void testReadWithSeveralUncommitted(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- // Put some data on the DB
- Transaction t = tm.begin();
- Put put = new Put(row);
- put.add(family, col, data);
- table.put(t, put);
- tm.commit(t);
- List<Transaction> running = new ArrayList<>();
-
- // Shade the data with uncommitted data
- for (int i = 0; i < 10; ++i) {
- t = tm.begin();
- put = new Put(row);
- put.add(family, col, uncommitted);
- table.put(t, put);
- running.add(t);
- }
-
- // Try to read from row, it should ignore the uncommitted data and return the original committed value
- t = tm.begin();
- Get get = new Get(row);
- Result result = table.get(t, get);
- Cell cell = result.getColumnLatestCell(family, col);
- assertNotNull("KeyValue is null", cell);
- byte[] value = CellUtil.cloneValue(cell);
- assertTrue("Read data doesn't match", Arrays.equals(data, value));
- tm.commit(t);
-
- table.close();
-
- for (Transaction r : running) {
- tm.rollback(r);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
deleted file mode 100644
index 7e9b406..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestShadowCells.java
+++ /dev/null
@@ -1,508 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Charsets;
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.yahoo.omid.committable.CommitTable;
-
-import com.yahoo.omid.metrics.NullMetricsProvider;
-
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.mockito.Matchers;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static com.yahoo.omid.transaction.CellUtils.hasCell;
-import static com.yahoo.omid.transaction.CellUtils.hasShadowCell;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestShadowCells extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestShadowCells.class);
-
- private static final String TSO_SERVER_HOST = "localhost";
- private static final int TSO_SERVER_PORT = 1234;
-
- private static final String TEST_TABLE = "test";
- private static final String TEST_FAMILY = "data";
-
- static final byte[] row = Bytes.toBytes("test-sc");
- private static final byte[] row1 = Bytes.toBytes("test-sc1");
- private static final byte[] row2 = Bytes.toBytes("test-sc2");
- private static final byte[] row3 = Bytes.toBytes("test-sc3");
- static final byte[] family = Bytes.toBytes(TEST_FAMILY);
- private static final byte[] qualifier = Bytes.toBytes("testdata");
- private static final byte[] data1 = Bytes.toBytes("testWrite-1");
-
-
- @Test(timeOut = 60_000)
- public void testShadowCellsBasics(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
-
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction t1 = (HBaseTransaction) tm.begin();
-
- // Test shadow cells are created properly
- Put put = new Put(row);
- put.add(family, qualifier, data1);
- table.put(t1, put);
-
- // Before commit test that only the cell is there
- assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Cell should be there");
- assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Shadow cell shouldn't be there");
-
- tm.commit(t1);
-
- // After commit test that both cell and shadow cell are there
- assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Cell should be there");
- assertTrue(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Shadow cell should be there");
-
- // Test that we can make a valid read after adding a shadow cell without hitting the commit table
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- TransactionManager tm2 = HBaseTransactionManager.builder(hbaseOmidClientConf)
- .commitTableClient(commitTableClient)
- .build();
-
- Transaction t2 = tm2.begin();
- Get get = new Get(row);
- get.addColumn(family, qualifier);
-
- Result getResult = table.get(t2, get);
- assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
- verify(commitTableClient, never()).getCommitTimestamp(anyLong());
- }
-
- @Test(timeOut = 60_000)
- public void testCrashingAfterCommitDoesNotWriteShadowCells(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- PostCommitActions syncPostCommitter = spy(
- new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
- .postCommitter(syncPostCommitter)
- .commitTableClient(commitTableClient)
- .build());
-
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction t1 = (HBaseTransaction) tm.begin();
-
- // Test shadow cell are created properly
- Put put = new Put(row);
- put.add(family, qualifier, data1);
- table.put(t1, put);
- try {
- tm.commit(t1);
- } catch (Exception e) { // (*) crash
- // Do nothing
- }
-
- // After commit with the emulated crash, test that only the cell is there
- assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Cell should be there");
- assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Shadow cell should not be there");
-
- Transaction t2 = tm.begin();
- Get get = new Get(row);
- get.addColumn(family, qualifier);
-
- Result getResult = table.get(t2, get);
- assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Shadow cell should not be there");
- verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
- }
-
- @Test(timeOut = 60_000)
- public void testShadowCellIsHealedAfterCommitCrash(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- PostCommitActions syncPostCommitter = spy(
- new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
- .postCommitter(syncPostCommitter)
- .commitTableClient(commitTableClient)
- .build());
-
- // The following line emulates a crash after commit that is observed in (*) below
- doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction t1 = (HBaseTransaction) tm.begin();
-
- // Test shadow cell are created properly
- Put put = new Put(row);
- put.add(family, qualifier, data1);
- table.put(t1, put);
- try {
- tm.commit(t1);
- } catch (Exception e) { // (*) Crash
- // Do nothing
- }
-
- assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Cell should be there");
- assertFalse(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Shadow cell should not be there");
-
- Transaction t2 = tm.begin();
- Get get = new Get(row);
- get.addColumn(family, qualifier);
-
- // This get should heal the shadow cell
- Result getResult = table.get(t2, get);
- assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
- verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
-
- assertTrue(hasCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Cell should be there");
- assertTrue(hasShadowCell(row, family, qualifier, t1.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Shadow cell should be there after being healed");
-
- // As the shadow cell is healed, this get shouldn't have to hit the storage,
- // so the number of invocations to commitTableClient.getCommitTimestamp()
- // should remain the same
- getResult = table.get(t2, get);
- assertTrue(Arrays.equals(data1, getResult.getValue(family, qualifier)), "Values should be the same");
- verify(commitTableClient, times(1)).getCommitTimestamp(anyLong());
- }
-
- @Test(timeOut = 60_000)
- public void testTransactionNeverCompletesWhenAnExceptionIsThrownUpdatingShadowCells(ITestContext context)
- throws Exception {
-
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- PostCommitActions syncPostCommitter = spy(
- new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- AbstractTransactionManager tm = spy((AbstractTransactionManager) HBaseTransactionManager.builder(hbaseOmidClientConf)
- .postCommitter(syncPostCommitter)
- .commitTableClient(commitTableClient)
- .build());
-
- final TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- HBaseTransaction tx = (HBaseTransaction) tm.begin();
-
- Put put = new Put(row);
- put.add(family, qualifier, data1);
- table.put(tx, put);
-
- // This line emulates an error accessing the target table by disabling it
- doAnswer(new Answer<ListenableFuture<Void>>() {
- @Override
- public ListenableFuture<Void> answer(InvocationOnMock invocation) throws Throwable {
- table.flushCommits();
- HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
- admin.disableTable(table.getTableName());
- return (ListenableFuture<Void>) invocation.callRealMethod();
- }
- }).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- // When committing, an IOException should be thrown in syncPostCommitter.updateShadowCells() and placed in the
- // future as a TransactionManagerException. However, the exception is never retrieved in the
- // AbstractTransactionManager as the future is never checked.
- // This requires to set the HConstants.HBASE_CLIENT_RETRIES_NUMBER in the HBase config to a finite number:
- // e.g -> hbaseConf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 3); Otherwise it will get stuck in tm.commit();
-
- tm.commit(tx); // Tx effectively commits but the post Commit Actions failed when updating the shadow cells
-
- // Re-enable table to allow the required checks below
- HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
- admin.enableTable(table.getTableName());
-
- // 1) check that shadow cell is not created...
- assertTrue(hasCell(row, family, qualifier, tx.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Cell should be there");
- assertFalse(hasShadowCell(row, family, qualifier, tx.getStartTimestamp(), new TTableCellGetterAdapter(table)),
- "Shadow cell should not be there");
- // 2) and thus, completeTransaction() was never called on the commit table...
- verify(commitTableClient, times(0)).completeTransaction(anyLong());
- // 3) ...and commit value still in commit table
- assertTrue(commitTableClient.getCommitTimestamp(tx.getStartTimestamp()).get().isPresent());
-
- }
-
- @Test(timeOut = 60_000)
- public void testRaceConditionBetweenReaderAndWriterThreads(final ITestContext context) throws Exception {
- final CountDownLatch readAfterCommit = new CountDownLatch(1);
- final CountDownLatch postCommitBegin = new CountDownLatch(1);
- final CountDownLatch postCommitEnd = new CountDownLatch(1);
-
- final AtomicBoolean readFailed = new AtomicBoolean(false);
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), getCommitTable(context).getClient()));
- AbstractTransactionManager tm = (AbstractTransactionManager) newTransactionManager(context, syncPostCommitter);
-
- doAnswer(new Answer<ListenableFuture<Void>>() {
- @Override
- public ListenableFuture<Void> answer(InvocationOnMock invocation) throws Throwable {
- LOG.info("Releasing readAfterCommit barrier");
- readAfterCommit.countDown();
- LOG.info("Waiting postCommitBegin barrier");
- postCommitBegin.await();
- ListenableFuture<Void> result = (ListenableFuture<Void>) invocation.callRealMethod();
- LOG.info("Releasing postCommitEnd barrier");
- postCommitEnd.countDown();
- return result;
- }
- }).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
-
- // Start transaction on write thread
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- final HBaseTransaction t1 = (HBaseTransaction) tm.begin();
-
- // Start read thread
- Thread readThread = new Thread("Read Thread") {
- @Override
- public void run() {
- LOG.info("Waiting readAfterCommit barrier");
- try {
- readAfterCommit.await();
- final TTable table = spy(new TTable(hbaseConf, TEST_TABLE));
- doAnswer(new Answer<List<KeyValue>>() {
- @SuppressWarnings("unchecked")
- @Override
- public List<KeyValue> answer(InvocationOnMock invocation) throws Throwable {
- LOG.info("Release postCommitBegin barrier");
- postCommitBegin.countDown();
- LOG.info("Waiting postCommitEnd barrier");
- postCommitEnd.await();
- return (List<KeyValue>) invocation.callRealMethod();
- }
- }).when(table).filterCellsForSnapshot(Matchers.<List<Cell>>any(),
- any(HBaseTransaction.class), anyInt());
-
- TransactionManager tm = newTransactionManager(context);
- if (hasShadowCell(row,
- family,
- qualifier,
- t1.getStartTimestamp(),
- new TTableCellGetterAdapter(table))) {
- readFailed.set(true);
- }
-
- Transaction t = tm.begin();
- Get get = new Get(row);
- get.addColumn(family, qualifier);
-
- Result getResult = table.get(t, get);
- Cell cell = getResult.getColumnLatestCell(family, qualifier);
- if (!Arrays.equals(data1, CellUtil.cloneValue(cell))
- || !hasShadowCell(row,
- family,
- qualifier,
- cell.getTimestamp(),
- new TTableCellGetterAdapter(table))) {
- readFailed.set(true);
- } else {
- LOG.info("Read succeeded");
- }
- } catch (Throwable e) {
- readFailed.set(true);
- LOG.error("Error whilst reading", e);
- }
- }
- };
- readThread.start();
-
- // Write data
- Put put = new Put(row);
- put.add(family, qualifier, data1);
- table.put(t1, put);
- tm.commit(t1);
-
- readThread.join();
-
- assertFalse(readFailed.get(), "Read should have succeeded");
-
- }
-
- // TODO: After removing the legacy shadow cell suffix, maybe we should mix the assertions in this test with
- // the ones in the previous tests in a further commit
-
- /**
- * Test that the new client can read shadow cells written by the old client.
- */
- @Test
- public void testGetOldShadowCells(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
-
- TTable table = new TTable(hbaseConf, TEST_TABLE);
- HTableInterface htable = table.getHTable();
-
- // Test shadow cell are created properly
- HBaseTransaction t1 = (HBaseTransaction) tm.begin();
- Put put = new Put(row1);
- put.add(family, qualifier, data1);
- table.put(t1, put);
- tm.commit(t1);
-
- HBaseTransaction t2 = (HBaseTransaction) tm.begin();
- put = new Put(row2);
- put.add(family, qualifier, data1);
- table.put(t2, put);
- tm.commit(t2);
-
- HBaseTransaction t3 = (HBaseTransaction) tm.begin();
- put = new Put(row3);
- put.add(family, qualifier, data1);
- table.put(t3, put);
- tm.commit(t3);
-
- // ensure that transaction is no longer in commit table
- // the only place that should have the mapping is the shadow cells
- CommitTable.Client commitTableClient = spy(getCommitTable(context).getClient());
- Optional<CommitTable.CommitTimestamp> ct1 = commitTableClient.getCommitTimestamp(t1.getStartTimestamp()).get();
- Optional<CommitTable.CommitTimestamp> ct2 = commitTableClient.getCommitTimestamp(t2.getStartTimestamp()).get();
- Optional<CommitTable.CommitTimestamp> ct3 = commitTableClient.getCommitTimestamp(t3.getStartTimestamp()).get();
- assertFalse(ct1.isPresent(), "Shouldn't exist in commit table");
- assertFalse(ct2.isPresent(), "Shouldn't exist in commit table");
- assertFalse(ct3.isPresent(), "Shouldn't exist in commit table");
-
- // delete new shadow cell
- Delete del = new Delete(row2);
- del.deleteColumn(family, CellUtils.addShadowCellSuffix(qualifier));
- htable.delete(del);
- htable.flushCommits();
-
- // verify that we can't read now (since shadow cell is missing)
- Transaction t4 = tm.begin();
- Get get = new Get(row2);
- get.addColumn(family, qualifier);
-
- Result getResult = table.get(t4, get);
- assertTrue(getResult.isEmpty(), "Should have nothing");
-
- Transaction t5 = tm.begin();
- Scan s = new Scan();
- ResultScanner scanner = table.getScanner(t5, s);
- Result result1 = scanner.next();
- Result result2 = scanner.next();
- Result result3 = scanner.next();
- scanner.close();
-
- assertNull(result3);
- assertTrue(Arrays.equals(result1.getRow(), row1), "Should have first row");
- assertTrue(Arrays.equals(result2.getRow(), row3), "Should have third row");
- assertTrue(result1.containsColumn(family, qualifier), "Should have column family");
- assertTrue(result2.containsColumn(family, qualifier), "Should have column family");
-
- // now add in the previous legacy shadow cell for that row
- put = new Put(row2);
- put.add(family,
- addLegacyShadowCellSuffix(qualifier),
- t2.getStartTimestamp(),
- Bytes.toBytes(t2.getCommitTimestamp()));
- htable.put(put);
-
- // we should NOT be able to read that row now, even though
- // it has a legacy shadow cell
- Transaction t6 = tm.begin();
- get = new Get(row2);
- get.addColumn(family, qualifier);
-
- getResult = table.get(t6, get);
- assertFalse(getResult.containsColumn(family, qualifier), "Should NOT have column");
-
- Transaction t7 = tm.begin();
- s = new Scan();
- scanner = table.getScanner(t7, s);
- result1 = scanner.next();
- result2 = scanner.next();
- result3 = scanner.next();
- scanner.close();
-
- assertNull(result3, "There should only be 2 rows");
- assertTrue(Arrays.equals(result1.getRow(), row1), "Should have first row");
- assertTrue(Arrays.equals(result2.getRow(), row3), "Should have third row");
- assertTrue(result1.containsColumn(family, qualifier), "Should have column family");
- assertTrue(result2.containsColumn(family, qualifier), "Should have column family");
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private static final byte[] LEGACY_SHADOW_CELL_SUFFIX = ":OMID_CTS".getBytes(Charsets.UTF_8);
-
- private static byte[] addLegacyShadowCellSuffix(byte[] qualifier) {
- return com.google.common.primitives.Bytes.concat(qualifier, LEGACY_SHADOW_CELL_SUFFIX);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
deleted file mode 100644
index b8b28fc..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestSingleColumnFamily.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.testng.AssertJUnit.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestSingleColumnFamily extends OmidTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(TestSingleColumnFamily.class);
-
-
- @Test
- public void testSingleColumnFamily(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable table1 = new TTable(hbaseConf, TEST_TABLE);
- int num = 10;
- Transaction t = tm.begin();
- for (int j = 0; j < num; j++) {
- byte[] data = Bytes.toBytes(j);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value1"), data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2"), data);
- table1.put(t, put);
- }
- //tm.tryCommit(t);
- //t=tm.beginTransaction(); //Visible if in a different transaction
- Scan s = new Scan();
- ResultScanner res = table1.getScanner(t, s);
- Result rr;
- int count = 0;
- while ((rr = res.next()) != null) {
- int tmp1 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value1")));
- int tmp2 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2")));
- LOG.info("RES:" + tmp1 + ";" + tmp2);
- count++;
- }
- assertTrue("Can't see puts. I should see "
- + num + " but I see " + count
- , num == count);
-
- tm.commit(t);
- t = tm.begin();
-
- for (int j = 0; j < num / 2; j++) {
- byte[] data = Bytes.toBytes(j);
- byte[] ndata = Bytes.toBytes(j * 10);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2"), ndata);
- table1.put(t, put);
- }
- tm.commit(t);
- t = tm.begin();
- s = new Scan();
- res = table1.getScanner(t, s);
- count = 0;
- int modified = 0, notmodified = 0;
- while ((rr = res.next()) != null) {
- int tmp1 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value1")));
- int tmp2 = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes("value2")));
- LOG.info("RES:" + tmp1 + ";" + tmp2);
- count++;
-
- if (tmp2 == Bytes.toInt(rr.getRow()) * 10) {
- modified++;
- } else {
- notmodified++;
- }
- if (count == 8) {
- LOG.debug("stop");
- }
- }
- assertTrue("Can't see puts. I should see "
- + num + " but I see " + count
- , num == count);
- assertTrue("Half of rows should equal row id, half not ("
- + modified + ", " + notmodified + ")"
- , modified == notmodified && notmodified == (num / 2));
-
- tm.commit(t);
- LOG.info("End commiting");
- table1.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
deleted file mode 100644
index dced462..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTSOModule.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorage;
-import com.yahoo.omid.timestamp.storage.TimestampStorage;
-import com.yahoo.omid.tso.DisruptorModule;
-import com.yahoo.omid.tso.MockPanicker;
-import com.yahoo.omid.tso.NetworkInterfaceUtils;
-import com.yahoo.omid.tso.Panicker;
-import com.yahoo.omid.tso.PausableTimestampOracle;
-import com.yahoo.omid.tso.TSOChannelHandler;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.TSOStateManager;
-import com.yahoo.omid.tso.TSOStateManagerImpl;
-import com.yahoo.omid.tso.TimestampOracle;
-import org.apache.hadoop.conf.Configuration;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-import java.net.SocketException;
-import java.net.UnknownHostException;
-
-import static com.yahoo.omid.tso.TSOServer.TSO_HOST_AND_PORT_KEY;
-
-class TestTSOModule extends AbstractModule {
-
- private final Configuration hBaseConfig;
- private final TSOServerConfig config;
-
- TestTSOModule(Configuration hBaseConfig, TSOServerConfig config) {
- this.hBaseConfig = hBaseConfig;
- this.config = config;
- }
-
- @Override
- protected void configure() {
-
- bind(TSOChannelHandler.class).in(Singleton.class);
-
- bind(TSOStateManager.class).to(TSOStateManagerImpl.class).in(Singleton.class);
-
- bind(CommitTable.class).to(HBaseCommitTable.class).in(Singleton.class);
- bind(TimestampStorage.class).to(HBaseTimestampStorage.class).in(Singleton.class);
- bind(TimestampOracle.class).to(PausableTimestampOracle.class).in(Singleton.class);
- bind(Panicker.class).to(MockPanicker.class).in(Singleton.class);
-
- // Disruptor setup
- install(new DisruptorModule());
-
- // LeaseManagement setup
- install(config.getLeaseModule());
- }
-
- @Provides
- Configuration provideHBaseConfig() {
- return hBaseConfig;
- }
-
- @Provides
- TSOServerConfig provideTSOServerConfig() {
- return config;
- }
-
- @Provides
- @Singleton
- MetricsRegistry provideMetricsRegistry() {
- return new NullMetricsProvider();
- }
-
- @Provides
- @Named(TSO_HOST_AND_PORT_KEY)
- String provideTSOHostAndPort() throws SocketException, UnknownHostException {
- return NetworkInterfaceUtils.getTSOHostAndPort(config);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
deleted file mode 100644
index 023ebc6..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTTableBehaviour.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Charsets;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.mockito.Mockito;
-import org.testng.annotations.Test;
-
-import static org.testng.Assert.fail;
-
-@Test(groups = "noHBase")
-public class TestTTableBehaviour {
-
- private byte[] row = Bytes.toBytes("1row");
- private byte[] famName = Bytes.toBytes("tf");
- private byte[] colName = Bytes.toBytes("tc");
- private byte[] dataValue = Bytes.toBytes("test-data");
-
- @Test(timeOut = 10_000)
- public void testUserOperationsDontAllowTimestampSpecification() throws Exception {
-
- // Component under test
- TTable tt = new TTable(Mockito.mock(HTableInterface.class), Mockito.mock(HTableInterface.class));
-
- long randomTimestampValue = Bytes.toLong("deadbeef".getBytes());
-
- Transaction tx = Mockito.mock(Transaction.class);
-
- // Test put fails when a timestamp is specified in the put
- Put put = new Put(row, randomTimestampValue);
- put.add(famName, colName, dataValue);
- try {
- tt.put(tx, put);
- fail("Should have thrown an IllegalArgumentException due to timestamp specification");
- } catch (IllegalArgumentException e) {
- // Continue
- }
-
- // Test put fails when a timestamp is specified in a qualifier
- put = new Put(row);
- put.add(famName, colName, randomTimestampValue, dataValue);
- try {
- tt.put(tx, put);
- fail("Should have thrown an IllegalArgumentException due to timestamp specification");
- } catch (IllegalArgumentException e) {
- // Continue
- }
-
- // Test that get fails when a timestamp is specified
- Get get = new Get(row);
- get.setTimeStamp(randomTimestampValue);
- try {
- tt.get(tx, get);
- fail("Should have thrown an IllegalArgumentException due to timestamp specification");
- } catch (IllegalArgumentException e) {
- // Continue
- }
-
- // Test scan fails when a timerange is specified
- Scan scan = new Scan(get);
- try {
- tt.getScanner(tx, scan);
- fail("Should have thrown an IllegalArgumentException due to timestamp specification");
- } catch (IllegalArgumentException e) {
- // Continue
- }
-
- // Test delete fails when a timestamp is specified
- Delete delete = new Delete(row);
- delete.setTimestamp(randomTimestampValue);
- try {
- tt.delete(tx, delete);
- fail("Should have thrown an IllegalArgumentException due to timestamp specification");
- } catch (IllegalArgumentException e) {
- // Continue
- }
-
- // Test delete fails when a timestamp is specified in a qualifier
- delete = new Delete(row);
- delete.deleteColumn(famName, colName, randomTimestampValue);
- try {
- tt.delete(tx, delete);
- fail("Should have thrown an IllegalArgumentException due to timestamp specification");
- } catch (IllegalArgumentException e) {
- // Continue
- }
-
- }
-
- /**
- * Test that we cannot use reserved names for shadow cell identifiers as qualifiers in user operations
- */
- @Test(timeOut = 10_000)
- public void testReservedNamesForShadowCellsCanNotBeUsedAsQualifiersInUserOperations() throws Exception {
- byte[] nonValidQualifier1 = "blahblah\u0080".getBytes(Charsets.UTF_8);
- byte[] validQualifierIncludingOldShadowCellSuffix = "blahblah:OMID_CTS".getBytes(Charsets.UTF_8);
-
- TTable table = new TTable(Mockito.mock(HTableInterface.class), Mockito.mock(HTableInterface.class));
-
- HBaseTransaction t1 = Mockito.mock(HBaseTransaction.class);
- Put put = new Put(row);
- put.add(famName, nonValidQualifier1, dataValue);
- try {
- table.put(t1, put);
- fail("Shouldn't be able to put this");
- } catch (IllegalArgumentException iae) {
- // correct
- }
- Delete del = new Delete(row);
- del.deleteColumn(famName, nonValidQualifier1);
- try {
- table.delete(t1, del);
- fail("Shouldn't be able to delete this");
- } catch (IllegalArgumentException iae) {
- // correct
- }
-
- put = new Put(row);
- put.add(famName, validQualifierIncludingOldShadowCellSuffix, dataValue);
- try {
- table.put(t1, put);
- } catch (IllegalArgumentException iae) {
- fail("Qualifier shouldn't be rejected anymore");
- }
- del = new Delete(row);
- del.deleteColumn(famName, validQualifierIncludingOldShadowCellSuffix);
- try {
- table.delete(t1, del);
- } catch (IllegalArgumentException iae) {
- fail("Qualifier shouldn't be rejected anymore");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
deleted file mode 100644
index f74f659..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionCleanup.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.tso.client.AbortException;
-import com.yahoo.omid.tso.client.ForwardingTSOFuture;
-import com.yahoo.omid.tso.client.TSOClient;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.mockito.Matchers.anySetOf;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.testng.AssertJUnit.assertEquals;
-
-@Test(groups = "sharedHBase")
-public class TestTransactionCleanup extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTransactionCleanup.class);
-
- private static final long START_TS = 1L;
-
- private byte[] row = Bytes.toBytes("row");
- private byte[] family = Bytes.toBytes(TEST_FAMILY);
- private byte[] qual = Bytes.toBytes("qual");
- private byte[] data = Bytes.toBytes("data");
-
- // NOTE: This test is maybe redundant with runTestCleanupAfterConflict()
- // and testCleanupWithDeleteRow() tests in TestTransactionCleanup class.
- // Code in TestTransactionCleanup is a little more difficult to follow,
- // lacks some assertions and includes some magic numbers, so we should
- // try to review and improve the tests in these two classes in a further
- // commit.
- @Test
- public void testTransactionIsCleanedUpAfterBeingAborted(ITestContext context) throws Exception {
-
- final int ROWS_MODIFIED = 1;
-
- // Prepare the mocking results
- SettableFuture<Long> startTSF = SettableFuture.create();
- startTSF.set(START_TS);
- ForwardingTSOFuture<Long> stFF = new ForwardingTSOFuture<>(startTSF);
-
- SettableFuture<Long> abortingF = SettableFuture.create();
- abortingF.setException(new AbortException());
- ForwardingTSOFuture<Long> abortingFF = new ForwardingTSOFuture<>(abortingF);
-
- // Mock the TSO Client setting the right method responses
- TSOClient mockedTSOClient = mock(TSOClient.class);
-
- doReturn(stFF)
- .when(mockedTSOClient).getNewStartTimestamp();
-
- doReturn(abortingFF)
- .when(mockedTSOClient).commit(eq(START_TS), anySetOf(HBaseCellId.class));
-
- try (TransactionManager tm = newTransactionManager(context, mockedTSOClient);
- TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Start a transaction and put some data in a column
- Transaction tx = tm.begin();
-
- Put put = new Put(row);
- put.add(family, qual, data);
- txTable.put(tx, put);
-
- // Abort transaction when committing, so the cleanup
- // process we want to test is triggered
- try {
- tm.commit(tx);
- } catch (RollbackException e) {
- // Expected
- }
-
- // So now we have to check that the Delete marker introduced by the
- // cleanup process is there
- Scan scan = new Scan(row);
- scan.setRaw(true); // Raw scan to obtain the deleted cells
- ResultScanner resultScanner = txTable.getHTable().getScanner(scan);
- int resultCount = 0;
- for (Result result : resultScanner) {
- assertEquals(2, result.size()); // Size == 2, including the put and delete from cleanup
- LOG.trace("Result {}", result);
- // The last element of the qualifier should have the Delete marker
- byte encodedType = result.getColumnLatestCell(family, qual).getTypeByte();
- assertEquals(KeyValue.Type.Delete,
- KeyValue.Type.codeToType(encodedType));
- resultCount++;
- }
- assertEquals(ROWS_MODIFIED, resultCount);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
deleted file mode 100644
index 627fe79..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTransactionConflict.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestTransactionConflict extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTransactionConflict.class);
-
-
- @Test
- public void runTestWriteWriteConflict(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- Transaction t2 = tm.begin();
- LOG.info("Transaction created" + t2);
-
- byte[] row = Bytes.toBytes("test-simple");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
-
- Put p2 = new Put(row);
- p2.add(fam, col, data2);
- tt.put(t2, p2);
-
- tm.commit(t2);
-
- try {
- tm.commit(t1);
- Assert.fail("Transaction should not commit successfully");
- } catch (RollbackException e) {
- }
- }
-
- @Test
- public void runTestMultiTableConflict(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
- String table2 = TEST_TABLE + 2;
- TableName table2Name = TableName.valueOf(table2);
-
- HBaseAdmin admin = new HBaseAdmin(hbaseConf);
-
- if (!admin.tableExists(table2)) {
- HTableDescriptor desc = new HTableDescriptor(table2Name);
- HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY);
- datafam.setMaxVersions(Integer.MAX_VALUE);
- desc.addFamily(datafam);
-
- admin.createTable(desc);
- }
-
- if (admin.isTableDisabled(table2)) {
- admin.enableTable(table2);
- }
- admin.close();
-
- TTable tt2 = new TTable(hbaseConf, table2);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- Transaction t2 = tm.begin();
- LOG.info("Transaction created" + t2);
-
- byte[] row = Bytes.toBytes("test-simple");
- byte[] row2 = Bytes.toBytes("test-simple2");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
- tt2.put(t1, p);
-
- Put p2 = new Put(row);
- p2.add(fam, col, data2);
- tt.put(t2, p2);
- p2 = new Put(row2);
- p2.add(fam, col, data2);
- tt2.put(t2, p2);
-
- tm.commit(t2);
-
- boolean aborted = false;
- try {
- tm.commit(t1);
- assertTrue("Transaction commited successfully", false);
- } catch (RollbackException e) {
- aborted = true;
- }
- assertTrue("Transaction didn't raise exception", aborted);
-
- ResultScanner rs = tt2.getHTable().getScanner(fam, col);
-
- int count = 0;
- Result r;
- while ((r = rs.next()) != null) {
- count += r.size();
- }
- assertEquals("Should have cell", 1, count);
- }
-
- @Test
- public void runTestCleanupAfterConflict(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- Transaction t2 = tm.begin();
- LOG.info("Transaction created" + t2);
-
- byte[] row = Bytes.toBytes("test-simple");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
-
- Get g = new Get(row).setMaxVersions();
- g.addColumn(fam, col);
- Result r = tt.getHTable().get(g);
- assertEquals("Unexpected size for read.", 1, r.size());
- assertTrue("Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)),
- Bytes.equals(data1, r.getValue(fam, col)));
-
- Put p2 = new Put(row);
- p2.add(fam, col, data2);
- tt.put(t2, p2);
-
- r = tt.getHTable().get(g);
- assertEquals("Unexpected size for read.", 2, r.size());
- r = tt.get(t2, g);
- assertEquals("Unexpected size for read.", 1, r.size());
- assertTrue("Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)),
- Bytes.equals(data2, r.getValue(fam, col)));
-
- tm.commit(t1);
-
- boolean aborted = false;
- try {
- tm.commit(t2);
- assertTrue("Transaction commited successfully", false);
- } catch (RollbackException e) {
- aborted = true;
- }
- assertTrue("Transaction didn't raise exception", aborted);
-
- r = tt.getHTable().get(g);
- assertEquals("Unexpected size for read.", 1, r.size());
- assertTrue("Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)),
- Bytes.equals(data1, r.getValue(fam, col)));
- }
-
- @Test
- public void testCleanupWithDeleteRow(ITestContext context) throws Exception {
- try {
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- int rowcount = 10;
- int count = 0;
-
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- byte[] modrow = Bytes.toBytes("test-del" + 3);
- for (int i = 0; i < rowcount; i++) {
- byte[] row = Bytes.toBytes("test-del" + i);
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
- }
- tm.commit(t1);
-
- Transaction t2 = tm.begin();
- LOG.info("Transaction created " + t2);
- Delete d = new Delete(modrow);
- tt.delete(t2, d);
-
- ResultScanner rs = tt.getScanner(t2, new Scan());
- Result r = rs.next();
- count = 0;
- while (r != null) {
- count++;
- LOG.trace("row: " + Bytes.toString(r.getRow()) + " count: " + count);
- r = rs.next();
- }
- assertEquals("Wrong count", rowcount - 1, count);
-
- Transaction t3 = tm.begin();
- LOG.info("Transaction created " + t3);
- Put p = new Put(modrow);
- p.add(fam, col, data2);
- tt.put(t3, p);
-
- tm.commit(t3);
-
- boolean aborted = false;
- try {
- tm.commit(t2);
- assertTrue("Didn't abort", false);
- } catch (RollbackException e) {
- aborted = true;
- }
- assertTrue("Didn't raise exception", aborted);
-
- Transaction tscan = tm.begin();
- rs = tt.getScanner(tscan, new Scan());
- r = rs.next();
- count = 0;
- while (r != null) {
- count++;
- r = rs.next();
- }
- assertEquals("Wrong count", rowcount, count);
-
- } catch (Exception e) {
- LOG.error("Exception occurred", e);
- throw e;
- }
- }
-
- @Test
- public void testMultipleCellChangesOnSameRow(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- Transaction t2 = tm.begin();
- LOG.info("Transactions created " + t1 + " " + t2);
-
- byte[] row = Bytes.toBytes("row");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col1 = Bytes.toBytes("testdata1");
- byte[] col2 = Bytes.toBytes("testdata2");
- byte[] data = Bytes.toBytes("testWrite-1");
-
- Put p2 = new Put(row);
- p2.add(fam, col1, data);
- tt.put(t2, p2);
- tm.commit(t2);
-
- Put p1 = new Put(row);
- p1.add(fam, col2, data);
- tt.put(t1, p1);
- tm.commit(t1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
deleted file mode 100644
index f26efc6..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestTxMgrFailover.java
+++ /dev/null
@@ -1,353 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.committable.InMemoryCommitTable;
-import com.yahoo.omid.transaction.Transaction.Status;
-import com.yahoo.omid.tso.ProgrammableTSOServer;
-import com.yahoo.omid.tso.client.TSOClient;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-
-import static com.yahoo.omid.committable.CommitTable.CommitTimestamp.Location.COMMIT_TABLE;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-@Test(groups = "sharedHBase")
-public class TestTxMgrFailover extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTxMgrFailover.class);
-
- private static final int TSO_SERVER_PORT = 3333;
- private static final String TSO_SERVER_HOST = "localhost";
-
- private static final long TX1_ST = 1L;
- private static final long TX1_CT = 2L;
-
- private static final byte[] qualifier = Bytes.toBytes("test-qual");
- private static final byte[] row1 = Bytes.toBytes("row1");
- private static final byte[] data1 = Bytes.toBytes("testWrite-1");
-
- // Used in test assertions
- private InMemoryCommitTable commitTable;
-
- private CommitTable.Client commitTableClient;
- private CommitTable.Writer commitTableWriter;
-
- // Allows to prepare the required responses to client request operations
- private ProgrammableTSOServer tso;
-
- // The transaction manager under test
- private HBaseTransactionManager tm;
-
- @BeforeClass(alwaysRun = true)
- public void beforeClass() throws Exception {
- // ------------------------------------------------------------------------------------------------------------
- // ProgrammableTSOServer setup
- // ------------------------------------------------------------------------------------------------------------
- tso = new ProgrammableTSOServer(TSO_SERVER_PORT);
- TestUtils.waitForSocketListening(TSO_SERVER_HOST, TSO_SERVER_PORT, 100);
- }
-
- @BeforeMethod(alwaysRun = true, timeOut = 30_000)
- public void beforeMethod() throws IOException, InterruptedException {
-
- commitTable = new InMemoryCommitTable(); // Use an in-memory commit table to speed up tests
- commitTableClient = spy(commitTable.getClient());
- commitTableWriter = spy(commitTable.getWriter());
-
- HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
- hbaseOmidClientConf.setConnectionString(TSO_SERVER_HOST + ":" + TSO_SERVER_PORT);
- hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
- TSOClient tsoClientForTM = spy(TSOClient.newInstance(hbaseOmidClientConf.getOmidClientConfiguration()));
-
- tm = spy(HBaseTransactionManager.builder(hbaseOmidClientConf)
- .tsoClient(tsoClientForTM)
- .commitTableClient(commitTableClient)
- .build());
- }
-
- @Test(timeOut = 30_000)
- public void testAbortResponseFromTSOThrowsRollbackExceptionInClient() throws Exception {
- // Program the TSO to return an ad-hoc Timestamp and an abort response for tx 1
- tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
- tso.queueResponse(new ProgrammableTSOServer.AbortResponse(TX1_ST));
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- assertEquals(tx1.getStartTimestamp(), TX1_ST);
- Put put = new Put(row1);
- put.add(TEST_FAMILY.getBytes(), qualifier, data1);
- txTable.put(tx1, put);
- assertEquals(hBaseUtils.countRows(new HTable(hbaseConf, TEST_TABLE)), 1, "Rows should be 1!");
- checkOperationSuccessOnCell(KeyValue.Type.Put, data1, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
- qualifier);
-
- try {
- tm.commit(tx1);
- fail();
- } catch (RollbackException e) {
- // Expected!
-
- }
-
- // Check transaction status
- assertEquals(tx1.getStatus(), Status.ROLLEDBACK);
- assertEquals(tx1.getCommitTimestamp(), 0);
- // Check the cleanup process did its job and the committed data is NOT there
- checkOperationSuccessOnCell(KeyValue.Type.Delete, null, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
- qualifier);
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testClientReceivesSuccessfulCommitForNonInvalidatedTxCommittedByPreviousTSO() throws Exception {
-
- // Program the TSO to return an ad-hoc Timestamp and an commit response with heuristic actions
- tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
- tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
- // Simulate that tx1 was committed by writing to commit table
- commitTableWriter.addCommittedTransaction(TX1_ST, TX1_CT);
- commitTableWriter.flush();
- assertEquals(commitTable.countElements(), 1, "Rows should be 1!");
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- assertEquals(tx1.getStartTimestamp(), TX1_ST);
- Put put = new Put(row1);
- put.add(TEST_FAMILY.getBytes(), qualifier, data1);
- txTable.put(tx1, put);
- // Should succeed
- tm.commit(tx1);
-
- // Check transaction status
- assertEquals(tx1.getStatus(), Status.COMMITTED);
- assertEquals(tx1.getCommitTimestamp(), TX1_CT);
- // Check the cleanup process did its job and the committed data is there
- // Note that now we do not clean up the commit table when exercising the heuristic actions
- assertEquals(commitTable.countElements(), 1,
- "Rows should be 1! We don't have to clean CT in this case");
- Optional<CommitTimestamp>
- optionalCT =
- tm.commitTableClient.getCommitTimestamp(TX1_ST).get();
- assertTrue(optionalCT.isPresent());
- checkOperationSuccessOnCell(KeyValue.Type.Put, data1, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
- qualifier);
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testClientReceivesRollbackExceptionForInvalidatedTxCommittedByPreviousTSO() throws Exception {
-
- // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
- tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
- tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
- // Simulate that tx1 was committed by writing to commit table but was later invalidated
- commitTableClient.tryInvalidateTransaction(TX1_ST);
- assertEquals(commitTable.countElements(), 1, "Rows should be 1!");
-
- executeTxAndCheckRollback();
-
- }
-
- @Test(timeOut = 30_000)
- public void testClientReceivesNotificationOfANewTSOCanInvalidateTransaction() throws Exception {
-
- // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
- tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
- tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
-
- assertEquals(commitTable.countElements(), 0, "Rows should be 0!");
-
- executeTxAndCheckRollback();
-
- }
-
- private void executeTxAndCheckRollback() throws IOException, TransactionException, InterruptedException, java.util.concurrent.ExecutionException {
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- assertEquals(tx1.getStartTimestamp(), TX1_ST);
- Put put = new Put(row1);
- put.add(TEST_FAMILY.getBytes(), qualifier, data1);
- txTable.put(tx1, put);
- try {
- tm.commit(tx1);
- fail();
- } catch (RollbackException e) {
- // Expected
- }
-
- // Check transaction status
- assertEquals(tx1.getStatus(), Status.ROLLEDBACK);
- assertEquals(tx1.getCommitTimestamp(), 0);
- // Check the cleanup process did its job and the uncommitted data is NOT there
- assertEquals(commitTable.countElements(), 1, "Rows should be 1! Dirty data should be there");
- Optional<CommitTimestamp>
- optionalCT =
- tm.commitTableClient.getCommitTimestamp(TX1_ST).get();
- assertTrue(optionalCT.isPresent());
- assertFalse(optionalCT.get().isValid());
- checkOperationSuccessOnCell(KeyValue.Type.Delete, null, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
- qualifier);
- }
- }
-
- @Test(timeOut = 30_000)
- public void testClientSuccessfullyCommitsWhenReceivingNotificationOfANewTSOAandCANTInvalidateTransaction()
- throws Exception {
-
- // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
- tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
- tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
-
- // Simulate that the original TSO was able to add the tx to commit table in the meantime
- commitTableWriter.addCommittedTransaction(TX1_ST, TX1_CT);
- commitTableWriter.flush();
- assertEquals(commitTable.countElements(), 1, "Rows should be 1!");
- SettableFuture<Optional<CommitTimestamp>> f1 = SettableFuture.create();
- f1.set(Optional.<CommitTimestamp>absent());
- SettableFuture<Optional<CommitTimestamp>> f2 = SettableFuture.create();
- f2.set(Optional.of(new CommitTimestamp(COMMIT_TABLE, TX1_CT, true)));
- doReturn(f1).doReturn(f2).when(commitTableClient).getCommitTimestamp(TX1_ST);
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- assertEquals(tx1.getStartTimestamp(), TX1_ST);
- Put put = new Put(row1);
- put.add(TEST_FAMILY.getBytes(), qualifier, data1);
- txTable.put(tx1, put);
-
- tm.commit(tx1);
-
- // Check transaction status
- assertEquals(tx1.getStatus(), Status.COMMITTED);
- assertEquals(tx1.getCommitTimestamp(), TX1_CT);
- // Check the cleanup process did its job and the committed data is there
- // Note that now we do not clean up the commit table when exercising the heuristic actions
- assertEquals(commitTable.countElements(), 1,
- "Rows should be 1! We don't have to clean CT in this case");
- checkOperationSuccessOnCell(KeyValue.Type.Put, data1, TEST_TABLE.getBytes(), row1, TEST_FAMILY.getBytes(),
- qualifier);
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testClientReceivesATransactionExceptionWhenReceivingNotificationOfANewTSOAndCANTInvalidateTransactionAndCTCheckIsUnsuccessful()
- throws Exception {
-
- // Program the TSO to return an ad-hoc Timestamp and a commit response with heuristic actions
- tso.queueResponse(new ProgrammableTSOServer.TimestampResponse(TX1_ST));
- tso.queueResponse(new ProgrammableTSOServer.CommitResponse(true, TX1_ST, TX1_CT));
-
- // Simulate that the original TSO was able to add the tx to commit table in the meantime
- SettableFuture<Boolean> f = SettableFuture.create();
- f.set(false);
- doReturn(f).when(commitTableClient).tryInvalidateTransaction(TX1_ST);
-
- assertEquals(commitTable.countElements(), 0, "Rows should be 0!");
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
- HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
- assertEquals(tx1.getStartTimestamp(), TX1_ST);
- Put put = new Put(row1);
- put.add(TEST_FAMILY.getBytes(), qualifier, data1);
- txTable.put(tx1, put);
- try {
- tm.commit(tx1);
- fail();
- } catch (TransactionException e) {
- // Expected but is not good because we're not able to determine the tx outcome
- }
-
- // Check transaction status
- assertEquals(tx1.getStatus(), Status.RUNNING);
- assertEquals(tx1.getCommitTimestamp(), 0);
- }
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- protected void checkOperationSuccessOnCell(KeyValue.Type targetOp,
- @Nullable byte[] expectedValue,
- byte[] tableName,
- byte[] row,
- byte[] fam,
- byte[] col) {
-
- try (HTable table = new HTable(hbaseConf, tableName)) {
- Get get = new Get(row).setMaxVersions(1);
- Result result = table.get(get);
- Cell latestCell = result.getColumnLatestCell(fam, col);
-
- switch (targetOp) {
- case Put:
- assertEquals(latestCell.getTypeByte(), targetOp.getCode());
- assertEquals(CellUtil.cloneValue(latestCell), expectedValue);
- LOG.trace("Value for " + Bytes.toString(tableName) + ":"
- + Bytes.toString(row) + ":" + Bytes.toString(fam) + ":"
- + Bytes.toString(col) + "=>" + Bytes.toString(CellUtil.cloneValue(latestCell))
- + " (" + Bytes.toString(expectedValue) + " expected)");
- break;
- case Delete:
- LOG.trace("Value for " + Bytes.toString(tableName) + ":"
- + Bytes.toString(row) + ":" + Bytes.toString(fam)
- + Bytes.toString(col) + " deleted");
- assertNull(latestCell);
- break;
- default:
- fail();
- }
- } catch (IOException e) {
- LOG.error("Error reading row " + Bytes.toString(tableName) + ":"
- + Bytes.toString(row) + ":" + Bytes.toString(fam)
- + Bytes.toString(col), e);
- fail();
- }
- }
-
-}
[10/52] [abbrv] incubator-omid git commit: Rename tsoclient package
to tso.client
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java
new file mode 100644
index 0000000..8f42353
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOFuture.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.Future;
+
+public interface TSOFuture<T> extends Future<T> {
+ public void addListener(Runnable listener, Executor executor);
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java
new file mode 100644
index 0000000..cb9e679
--- /dev/null
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/client/TSOProtocol.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import java.util.Set;
+
+/**
+ * Defines the protocol used on the client side to abstract communication to the TSO server
+ */
+public interface TSOProtocol {
+
+ /**
+ * Returns a new timestamp assigned by on the server-side
+ * @return the newly assigned timestamp as a future. If an error was detected, the future will contain a
+ * corresponding protocol exception
+ * @see TimestampOracle
+ * @see TSOServer
+ */
+ TSOFuture<Long> getNewStartTimestamp();
+
+ /**
+ * Returns the result of the conflict detection made on the server-side for the specified transaction
+ * @param transactionId
+ * the transaction to check for conflicts
+ * @param writeSet
+ * the writeSet of the transaction, which includes all the modified cells
+ * @return the commit timestamp as a future if the transaction was committed. If the transaction was aborted due
+ * to conflicts with a concurrent transaction, the future will include an AbortException. If an error was detected,
+ * the future will contain a corresponding protocol exception
+ * @see TimestampOracle
+ * @see TSOServer
+ */
+ TSOFuture<Long> commit(long transactionId, Set<? extends CellId> writeSet);
+
+ /**
+ * Closes the communication with the TSO server
+ * @return nothing. If an error was detected, the future will contain a corresponding protocol exception
+ */
+ TSOFuture<Void> close();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java b/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
index 600dc51..d4421e0 100644
--- a/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
+++ b/transaction-client/src/main/java/com/yahoo/omid/tso/util/DummyCellIdImpl.java
@@ -17,7 +17,7 @@
*/
package com.yahoo.omid.tso.util;
-import com.yahoo.omid.tsoclient.CellId;
+import com.yahoo.omid.tso.client.CellId;
public class DummyCellIdImpl implements CellId {
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
deleted file mode 100644
index 5a4d43c..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/AbortException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-/**
- * Thrown when the TSO server has aborted a transaction
- */
-public class AbortException extends Exception {
-
- private static final long serialVersionUID = 1861474360100681040L;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
deleted file mode 100644
index eab757e..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/CellId.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-public interface CellId {
-
- public long getCellId();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
deleted file mode 100644
index d257deb..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ClosingException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-/**
- * Thrown when an error is produced when performing the actions required
- * to close the communication with the TSO server
- */
-public class ClosingException extends Exception {
-
- private static final long serialVersionUID = -5681694952053689884L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
deleted file mode 100644
index e20dd06..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ConnectionException.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import java.io.IOException;
-
-/**
- * Thrown when there are problems with the comm channel with the TSO server
- * (e.g. when it is closed or disconnected)
- */
-public class ConnectionException extends IOException {
-
- private static final long serialVersionUID = -8489539195700267443L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
deleted file mode 100644
index a2b889e..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ForwardingTSOFuture.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.util.concurrent.ListenableFuture;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-public class ForwardingTSOFuture<T> implements TSOFuture<T> {
- private final ListenableFuture<T> future;
-
- public ForwardingTSOFuture(ListenableFuture<T> future) {
- this.future = future;
- }
-
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- return future.cancel(mayInterruptIfRunning);
- }
-
- @Override
- public boolean isCancelled() {
- return future.isCancelled();
- }
-
- @Override
- public boolean isDone() {
- return future.isDone();
- }
-
- @Override
- public T get() throws InterruptedException, ExecutionException {
- return future.get();
- }
-
- @Override
- public T get(long timeout, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- return future.get(timeout, unit);
- }
-
- @Override
- public void addListener(Runnable listener, Executor executor) {
- future.addListener(listener, executor);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
deleted file mode 100644
index dc28656..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/HandshakeFailedException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-/**
- * Thrown when some incompatibilities between the TSO client & server are
- * found
- */
-public class HandshakeFailedException extends Exception {
-
- private static final long serialVersionUID = 8545505066920548834L;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
deleted file mode 100644
index a3e5653..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/MockTSOClient.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable;
-
-import java.io.IOException;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class MockTSOClient implements TSOProtocol {
- private final AtomicLong timestampGenerator = new AtomicLong();
- private final int CONFLICT_MAP_SIZE = 1 * 1000 * 1000;
- private final long[] conflictMap = new long[CONFLICT_MAP_SIZE];
- private final AtomicLong lwm = new AtomicLong();
-
- private final CommitTable.Writer commitTable;
-
- public MockTSOClient(CommitTable.Writer commitTable) {
- this.commitTable = commitTable;
- }
-
- @Override
- public TSOFuture<Long> getNewStartTimestamp() {
- synchronized (conflictMap) {
- SettableFuture<Long> f = SettableFuture.<Long>create();
- f.set(timestampGenerator.incrementAndGet());
- return new ForwardingTSOFuture<Long>(f);
- }
- }
-
- @Override
- public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
- synchronized (conflictMap) {
- SettableFuture<Long> f = SettableFuture.<Long>create();
- if (transactionId < lwm.get()) {
- f.setException(new AbortException());
- return new ForwardingTSOFuture<Long>(f);
- }
-
- boolean canCommit = true;
- for (CellId c : cells) {
- int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
- if (conflictMap[index] >= transactionId) {
- canCommit = false;
- break;
- }
- }
-
- if (canCommit) {
- long commitTimestamp = timestampGenerator.incrementAndGet();
- for (CellId c : cells) {
- int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
- long oldVal = conflictMap[index];
- conflictMap[index] = commitTimestamp;
- long curLwm = lwm.get();
- while (oldVal > curLwm) {
- if (lwm.compareAndSet(curLwm, oldVal)) {
- break;
- }
- curLwm = lwm.get();
- }
- }
-
- f.set(commitTimestamp);
- try {
- commitTable.addCommittedTransaction(transactionId, commitTimestamp);
- commitTable.updateLowWatermark(lwm.get());
- commitTable.flush();
- } catch (IOException ioe) {
- f.setException(ioe);
- }
- } else {
- f.setException(new AbortException());
- }
- return new ForwardingTSOFuture<Long>(f);
- }
- }
-
- @Override
- public TSOFuture<Void> close() {
- SettableFuture<Void> f = SettableFuture.<Void>create();
- f.set(null);
- return new ForwardingTSOFuture<Void>(f);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
deleted file mode 100644
index 735f51c..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/NewTSOException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-/**
- * Thrown when a new TSO has been detected
- */
-public class NewTSOException extends Exception {
-
- private static final long serialVersionUID = -3250655858200759321L;
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
deleted file mode 100644
index 566964a..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/OmidClientConfiguration.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
-import com.yahoo.omid.YAMLUtils;
-
-/**
- * Configuration for Omid client side
- */
-public class OmidClientConfiguration {
-
- private static final String DEFAULT_CONFIG_FILE_NAME = "omid-client-config.yml";
-
- public enum ConnType {DIRECT, HA}
-
- public enum PostCommitMode {SYNC, ASYNC}
-
- // Basic connection related params
-
- private ConnType connectionType = ConnType.DIRECT;
- private String connectionString;
- private String zkCurrentTsoPath;
- private String zkNamespace;
- private int zkConnectionTimeoutInSecs;
-
- // Communication protocol related params
-
- private int requestMaxRetries;
- private int requestTimeoutInMs;
- private int reconnectionDelayInSecs;
- private int retryDelayInMs;
- private int executorThreads;
-
- // Transaction Manager related params
-
- private PostCommitMode postCommitMode = PostCommitMode.SYNC;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Instantiation
- // ----------------------------------------------------------------------------------------------------------------
-
- public OmidClientConfiguration() {
- new YAMLUtils().loadSettings(DEFAULT_CONFIG_FILE_NAME, this);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters and setters for config params
- // ----------------------------------------------------------------------------------------------------------------
-
- public ConnType getConnectionType() {
- return connectionType;
- }
-
- @Inject(optional = true)
- @Named("omid.client.connectionType")
- public void setConnectionType(ConnType connectionType) {
- this.connectionType = connectionType;
- }
-
- public String getConnectionString() {
- return connectionString;
- }
-
- @Inject(optional = true)
- @Named("omid.client.connectionString")
- public void setConnectionString(String connectionString) {
- this.connectionString = connectionString;
- }
-
- public int getZkConnectionTimeoutInSecs() {
- return zkConnectionTimeoutInSecs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.zkConnectionTimeoutInSecs")
- public void setZkConnectionTimeoutInSecs(int zkConnectionTimeoutInSecs) {
- this.zkConnectionTimeoutInSecs = zkConnectionTimeoutInSecs;
- }
-
- public int getRequestMaxRetries() {
- return requestMaxRetries;
- }
-
- @Inject(optional = true)
- @Named("omid.client.requestMaxRetries")
- public void setRequestMaxRetries(int requestMaxRetries) {
- this.requestMaxRetries = requestMaxRetries;
- }
-
- public int getRequestTimeoutInMs() {
- return requestTimeoutInMs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.requestTimeoutInMs")
- public void setRequestTimeoutInMs(int requestTimeoutInMs) {
- this.requestTimeoutInMs = requestTimeoutInMs;
- }
-
- public int getReconnectionDelayInSecs() {
- return reconnectionDelayInSecs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.reconnectionDelayInSecs")
- public void setReconnectionDelayInSecs(int reconnectionDelayInSecs) {
- this.reconnectionDelayInSecs = reconnectionDelayInSecs;
- }
-
- public int getRetryDelayInMs() {
- return retryDelayInMs;
- }
-
- @Inject(optional = true)
- @Named("omid.client.retryDelayInMs")
- public void setRetryDelayInMs(int retryDelayInMs) {
- this.retryDelayInMs = retryDelayInMs;
- }
-
- public int getExecutorThreads() {
- return executorThreads;
- }
-
- @Inject(optional = true)
- @Named("omid.client.executorThreads")
- public void setExecutorThreads(int executorThreads) {
- this.executorThreads = executorThreads;
- }
-
- public String getZkCurrentTsoPath() {
- return zkCurrentTsoPath;
- }
-
- @Inject(optional = true)
- @Named("omid.ha.zkCurrentTsoPath")
- public void setZkCurrentTsoPath(String zkCurrentTsoPath) {
- this.zkCurrentTsoPath = zkCurrentTsoPath;
- }
-
- public String getZkNamespace() {
- return zkNamespace;
- }
-
- @Inject(optional = true)
- @Named("omid.ha.zkNamespace")
- public void setZkNamespace(String zkNamespace) {
- this.zkNamespace = zkNamespace;
- }
-
- public PostCommitMode getPostCommitMode() {
- return postCommitMode;
- }
-
- @Inject(optional = true)
- @Named("omid.tm.postCommitMode")
- public void setPostCommitMode(PostCommitMode postCommitMode) {
- this.postCommitMode = postCommitMode;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
deleted file mode 100644
index 2f8717e..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/ServiceUnavailableException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-/**
- * Thrown when the requests from TSO client to the TSO server have reached
- * a number of retries
- */
-public class ServiceUnavailableException extends Exception {
-
- private static final long serialVersionUID = -1551974284011474385L;
-
- public ServiceUnavailableException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
deleted file mode 100644
index dcfa6ca..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOClient.java
+++ /dev/null
@@ -1,933 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.base.Charsets;
-import com.google.common.net.HostAndPort;
-import com.google.common.util.concurrent.AbstractFuture;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.proto.TSOProto;
-import com.yahoo.omid.zk.ZKUtils;
-import com.yahoo.statemachine.StateMachine;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.recipes.cache.ChildData;
-import org.apache.curator.framework.recipes.cache.NodeCache;
-import org.apache.curator.framework.recipes.cache.NodeCacheListener;
-import org.jboss.netty.bootstrap.ClientBootstrap;
-import org.jboss.netty.channel.Channel;
-import org.jboss.netty.channel.ChannelFactory;
-import org.jboss.netty.channel.ChannelFuture;
-import org.jboss.netty.channel.ChannelFutureListener;
-import org.jboss.netty.channel.ChannelHandlerContext;
-import org.jboss.netty.channel.ChannelPipeline;
-import org.jboss.netty.channel.ChannelStateEvent;
-import org.jboss.netty.channel.ExceptionEvent;
-import org.jboss.netty.channel.MessageEvent;
-import org.jboss.netty.channel.SimpleChannelHandler;
-import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
-import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
-import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
-import org.jboss.netty.handler.codec.protobuf.ProtobufDecoder;
-import org.jboss.netty.handler.codec.protobuf.ProtobufEncoder;
-import org.jboss.netty.util.HashedWheelTimer;
-import org.jboss.netty.util.Timeout;
-import org.jboss.netty.util.TimerTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayDeque;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Describes the abstract methods to communicate to the TSO server
- */
-public class TSOClient implements TSOProtocol, NodeCacheListener {
-
- private static final Logger LOG = LoggerFactory.getLogger(TSOClient.class);
-
- // Basic configuration constants & defaults TODO: Move DEFAULT_ZK_CLUSTER to a conf class???
- public static final String DEFAULT_ZK_CLUSTER = "localhost:2181";
-
- private static final long DEFAULT_EPOCH = -1L;
- private volatile long epoch = DEFAULT_EPOCH;
-
- // Attributes
- private CuratorFramework zkClient;
- private NodeCache currentTSOZNode;
-
- private ChannelFactory factory;
- private ClientBootstrap bootstrap;
- private Channel currentChannel;
- private final ScheduledExecutorService fsmExecutor;
- StateMachine.Fsm fsm;
-
- private final int requestTimeoutInMs;
- private final int requestMaxRetries;
- private final int tsoReconnectionDelayInSecs;
- private InetSocketAddress tsoAddr;
- private String zkCurrentTsoPath;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Construction
- // ----------------------------------------------------------------------------------------------------------------
-
- public static TSOClient newInstance(OmidClientConfiguration tsoClientConf)
- throws IOException, InterruptedException {
- return new TSOClient(tsoClientConf);
- }
-
- // Avoid instantiation
- private TSOClient(OmidClientConfiguration omidConf) throws IOException, InterruptedException {
-
- // Start client with Nb of active threads = 3 as maximum.
- int tsoExecutorThreads = omidConf.getExecutorThreads();
-
- factory = new NioClientSocketChannelFactory(
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-boss-%d").build()),
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("tsoclient-worker-%d").build()), tsoExecutorThreads);
- // Create the bootstrap
- bootstrap = new ClientBootstrap(factory);
-
- requestTimeoutInMs = omidConf.getRequestTimeoutInMs();
- requestMaxRetries = omidConf.getRequestMaxRetries();
- tsoReconnectionDelayInSecs = omidConf.getReconnectionDelayInSecs();
-
- LOG.info("Connecting to TSO...");
- HostAndPort hp;
- switch (omidConf.getConnectionType()) {
- case HA:
- zkClient = ZKUtils.initZKClient(omidConf.getConnectionString(),
- omidConf.getZkNamespace(),
- omidConf.getZkConnectionTimeoutInSecs());
- zkCurrentTsoPath = omidConf.getZkCurrentTsoPath();
- configureCurrentTSOServerZNodeCache(zkCurrentTsoPath);
- String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
- // TSO info includes the new TSO host:port address and epoch
- String[] currentTSOAndEpochArray = tsoInfo.split("#");
- hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
- setTSOAddress(hp.getHostText(), hp.getPort());
- epoch = Long.parseLong(currentTSOAndEpochArray[1]);
- LOG.info("\t* Current TSO host:port found in ZK: {} Epoch {}", hp, getEpoch());
- break;
- case DIRECT:
- default:
- hp = HostAndPort.fromString(omidConf.getConnectionString());
- setTSOAddress(hp.getHostText(), hp.getPort());
- LOG.info("\t* TSO host:port {} will be connected directly", hp);
- break;
- }
-
- fsmExecutor = Executors.newSingleThreadScheduledExecutor(
- new ThreadFactoryBuilder().setNameFormat("tsofsm-%d").build());
- fsm = new StateMachine.FsmImpl(fsmExecutor);
- fsm.setInitState(new DisconnectedState(fsm));
-
- ChannelPipeline pipeline = bootstrap.getPipeline();
- pipeline.addLast("lengthbaseddecoder", new LengthFieldBasedFrameDecoder(8 * 1024, 0, 4, 0, 4));
- pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
- pipeline.addLast("protobufdecoder", new ProtobufDecoder(TSOProto.Response.getDefaultInstance()));
- pipeline.addLast("protobufencoder", new ProtobufEncoder());
- pipeline.addLast("handler", new Handler(fsm));
-
- bootstrap.setOption("tcpNoDelay", true);
- bootstrap.setOption("keepAlive", true);
- bootstrap.setOption("reuseAddress", true);
- bootstrap.setOption("connectTimeoutMillis", 100);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // TSOProtocol interface
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * @see TSOProtocol#getNewStartTimestamp()
- */
- @Override
- public TSOFuture<Long> getNewStartTimestamp() {
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.TimestampRequest.Builder tsreqBuilder = TSOProto.TimestampRequest.newBuilder();
- builder.setTimestampRequest(tsreqBuilder.build());
- RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
- fsm.sendEvent(request);
- return new ForwardingTSOFuture<>(request);
- }
-
- /**
- * @see TSOProtocol#commit(long, Set)
- */
- @Override
- public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitbuilder = TSOProto.CommitRequest.newBuilder();
- commitbuilder.setStartTimestamp(transactionId);
- for (CellId cell : cells) {
- commitbuilder.addCellId(cell.getCellId());
- }
- builder.setCommitRequest(commitbuilder.build());
- RequestEvent request = new RequestEvent(builder.build(), requestMaxRetries);
- fsm.sendEvent(request);
- return new ForwardingTSOFuture<>(request);
- }
-
- /**
- * @see TSOProtocol#close()
- */
- @Override
- public TSOFuture<Void> close() {
- final CloseEvent closeEvent = new CloseEvent();
- fsm.sendEvent(closeEvent);
- closeEvent.addListener(new Runnable() {
- @Override
- public void run() {
- try {
- closeEvent.get();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- e.printStackTrace();
- } catch (ExecutionException e) {
- e.printStackTrace();
- } finally {
- fsmExecutor.shutdown();
- if (currentTSOZNode != null) {
- try {
- currentTSOZNode.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- if (zkClient != null) {
- zkClient.close();
- }
- }
-
- }
- }, fsmExecutor);
- return new ForwardingTSOFuture<>(closeEvent);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // High availability related interface
- // ----------------------------------------------------------------------------------------------------------------
-
- /**
- * Returns the epoch of the TSO server that initialized this transaction.
- * Used for high availability support.
- */
- public long getEpoch() {
- return epoch;
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // NodeCacheListener interface
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public void nodeChanged() throws Exception {
-
- String tsoInfo = getCurrentTSOInfoFoundInZK(zkCurrentTsoPath);
- // TSO info includes the new TSO host:port address and epoch
- String[] currentTSOAndEpochArray = tsoInfo.split("#");
- HostAndPort hp = HostAndPort.fromString(currentTSOAndEpochArray[0]);
- setTSOAddress(hp.getHostText(), hp.getPort());
- epoch = Long.parseLong(currentTSOAndEpochArray[1]);
- LOG.info("CurrentTSO ZNode changed. New TSO Host & Port {}/Epoch {}", hp, getEpoch());
- if (currentChannel != null && currentChannel.isConnected()) {
- LOG.info("\tClosing channel with previous TSO {}", currentChannel);
- currentChannel.close();
- }
-
- }
-
- // ****************************************** Finite State Machine ************************************************
-
- // ----------------------------------------------------------------------------------------------------------------
- // FSM: Events
- // ----------------------------------------------------------------------------------------------------------------
-
- private static class ParamEvent<T> implements StateMachine.Event {
-
- final T param;
-
- ParamEvent(T param) {
- this.param = param;
- }
-
- T getParam() {
- return param;
- }
- }
-
- private static class ErrorEvent extends ParamEvent<Throwable> {
-
- ErrorEvent(Throwable t) {
- super(t);
- }
- }
-
- private static class ConnectedEvent extends ParamEvent<Channel> {
-
- ConnectedEvent(Channel c) {
- super(c);
- }
- }
-
- private static class UserEvent<T> extends AbstractFuture<T>
- implements StateMachine.DeferrableEvent {
-
- void success(T value) {
- set(value);
- }
-
- @Override
- public void error(Throwable t) {
- setException(t);
- }
- }
-
- private static class CloseEvent extends UserEvent<Void> {
-
- }
-
- private static class ChannelClosedEvent extends ParamEvent<Throwable> {
-
- ChannelClosedEvent(Throwable t) {
- super(t);
- }
- }
-
- private static class ReconnectEvent implements StateMachine.Event {
-
- }
-
- private static class HandshakeTimeoutEvent implements StateMachine.Event {
-
- }
-
- private static class TimestampRequestTimeoutEvent implements StateMachine.Event {
-
- }
-
- private static class CommitRequestTimeoutEvent implements StateMachine.Event {
-
- final long startTimestamp;
-
- CommitRequestTimeoutEvent(long startTimestamp) {
- this.startTimestamp = startTimestamp;
- }
-
- public long getStartTimestamp() {
- return startTimestamp;
- }
- }
-
- private static class RequestEvent extends UserEvent<Long> {
-
- TSOProto.Request req;
- int retriesLeft;
-
- RequestEvent(TSOProto.Request req, int retriesLeft) {
- this.req = req;
- this.retriesLeft = retriesLeft;
- }
-
- TSOProto.Request getRequest() {
- return req;
- }
-
- void setRequest(TSOProto.Request request) {
- this.req = request;
- }
-
- int getRetriesLeft() {
- return retriesLeft;
- }
-
- void decrementRetries() {
- retriesLeft--;
- }
-
- }
-
- private static class ResponseEvent extends ParamEvent<TSOProto.Response> {
-
- ResponseEvent(TSOProto.Response r) {
- super(r);
- }
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // FSM: States
- // ----------------------------------------------------------------------------------------------------------------
-
- class BaseState extends StateMachine.State {
-
- BaseState(StateMachine.Fsm fsm) {
- super(fsm);
- }
-
- public StateMachine.State handleEvent(StateMachine.Event e) {
- LOG.error("Unhandled event {} while in state {}", e, this.getClass().getName());
- return this;
- }
- }
-
- class DisconnectedState extends BaseState {
-
- DisconnectedState(StateMachine.Fsm fsm) {
- super(fsm);
- LOG.debug("NEW STATE: DISCONNECTED");
- }
-
- public StateMachine.State handleEvent(RequestEvent e) {
- fsm.deferEvent(e);
- return tryToConnectToTSOServer();
- }
-
- public StateMachine.State handleEvent(CloseEvent e) {
- factory.releaseExternalResources();
- e.success(null);
- return this;
- }
-
- private StateMachine.State tryToConnectToTSOServer() {
- final InetSocketAddress tsoAddress = getAddress();
- LOG.info("Trying to connect to TSO [{}]", tsoAddress);
- ChannelFuture channelFuture = bootstrap.connect(tsoAddress);
- channelFuture.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture channelFuture) throws Exception {
- if (channelFuture.isSuccess()) {
- LOG.info("Connection to TSO [{}] established. Channel {}",
- tsoAddress, channelFuture.getChannel());
- } else {
- LOG.error("Failed connection attempt to TSO [{}] failed. Channel {}",
- tsoAddress, channelFuture.getChannel());
- }
- }
- });
- return new ConnectingState(fsm);
- }
- }
-
- private class ConnectingState extends BaseState {
-
- ConnectingState(StateMachine.Fsm fsm) {
- super(fsm);
- LOG.debug("NEW STATE: CONNECTING");
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- fsm.deferEvent(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ConnectedEvent e) {
- return new HandshakingState(fsm, e.getParam());
- }
-
- public StateMachine.State handleEvent(ChannelClosedEvent e) {
- return new ConnectionFailedState(fsm, e.getParam());
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- return new ConnectionFailedState(fsm, e.getParam());
- }
-
- }
-
- private static class RequestAndTimeout {
-
- final RequestEvent event;
- final Timeout timeout;
-
- RequestAndTimeout(RequestEvent event, Timeout timeout) {
- this.event = event;
- this.timeout = timeout;
- }
-
- RequestEvent getRequest() {
- return event;
- }
-
- Timeout getTimeout() {
- return timeout;
- }
-
- public String toString() {
- String info = "Request type ";
- if (event.getRequest().hasTimestampRequest()) {
- info += "[Timestamp]";
- } else if (event.getRequest().hasCommitRequest()) {
- info += "[Commit] Start TS ->" + event.getRequest().getCommitRequest().getStartTimestamp();
- } else {
- info += "NONE";
- }
- return info;
- }
- }
-
- private class HandshakingState extends BaseState {
-
- final Channel channel;
-
- final HashedWheelTimer timeoutExecutor = new HashedWheelTimer(
- new ThreadFactoryBuilder().setNameFormat("tso-client-timeout").build());
- final Timeout timeout;
-
- HandshakingState(StateMachine.Fsm fsm, Channel channel) {
- super(fsm);
- LOG.debug("NEW STATE: HANDSHAKING");
- this.channel = channel;
- TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
- // Add the required handshake capabilities when necessary
- handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
- channel.write(TSOProto.Request.newBuilder().setHandshakeRequest(handshake.build()).build());
- timeout = newTimeout();
- }
-
- private Timeout newTimeout() {
- if (requestTimeoutInMs > 0) {
- return timeoutExecutor.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) {
- fsm.sendEvent(new HandshakeTimeoutEvent());
- }
- }, 30, TimeUnit.SECONDS);
- } else {
- return null;
- }
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- fsm.deferEvent(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ResponseEvent e) {
- if (e.getParam().hasHandshakeResponse() && e.getParam().getHandshakeResponse().getClientCompatible()) {
- if (timeout != null) {
- timeout.cancel();
- }
- return new ConnectedState(fsm, channel, timeoutExecutor);
- } else {
- cleanupState();
- LOG.error("Client incompatible with server");
- return new HandshakeFailedState(fsm, new HandshakeFailedException());
- }
- }
-
- public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
- cleanupState();
- return new ClosingState(fsm);
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- cleanupState();
- Throwable exception = e.getParam();
- LOG.error("Error during handshake", exception);
- return new HandshakeFailedState(fsm, exception);
- }
-
- private void cleanupState() {
- timeoutExecutor.stop();
- channel.close();
- if (timeout != null) {
- timeout.cancel();
- }
- }
-
- }
-
- class ConnectionFailedState extends BaseState {
-
- final HashedWheelTimer reconnectionTimeoutExecutor = new HashedWheelTimer(
- new ThreadFactoryBuilder().setNameFormat("tso-client-backoff-timeout").build());
-
- Throwable exception;
-
- ConnectionFailedState(final StateMachine.Fsm fsm, final Throwable exception) {
- super(fsm);
- LOG.debug("NEW STATE: CONNECTION FAILED [RE-CONNECTION BACKOFF]");
- this.exception = exception;
- reconnectionTimeoutExecutor.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) {
- fsm.sendEvent(new ReconnectEvent());
- }
- }, tsoReconnectionDelayInSecs, TimeUnit.SECONDS);
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- e.error(exception);
- return this;
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- return this;
- }
-
- public StateMachine.State handleEvent(ChannelClosedEvent e) {
- return new DisconnectedState(fsm);
- }
-
- public StateMachine.State handleEvent(ReconnectEvent e) {
- return new DisconnectedState(fsm);
- }
-
- }
-
- private class HandshakeFailedState extends ConnectionFailedState {
-
- HandshakeFailedState(StateMachine.Fsm fsm, Throwable exception) {
- super(fsm, exception);
- LOG.debug("STATE: HANDSHAKING FAILED");
- }
-
- }
-
- class ConnectedState extends BaseState {
-
- final Queue<RequestAndTimeout> timestampRequests;
- final Map<Long, RequestAndTimeout> commitRequests;
- final Channel channel;
-
- final HashedWheelTimer timeoutExecutor;
-
- ConnectedState(StateMachine.Fsm fsm, Channel channel, HashedWheelTimer timeoutExecutor) {
- super(fsm);
- LOG.debug("NEW STATE: CONNECTED");
- this.channel = channel;
- this.timeoutExecutor = timeoutExecutor;
- timestampRequests = new ArrayDeque<>();
- commitRequests = new HashMap<>();
- }
-
- private Timeout newTimeout(final StateMachine.Event timeoutEvent) {
- if (requestTimeoutInMs > 0) {
- return timeoutExecutor.newTimeout(new TimerTask() {
- @Override
- public void run(Timeout timeout) {
- fsm.sendEvent(timeoutEvent);
- }
- }, requestTimeoutInMs, TimeUnit.MILLISECONDS);
- } else {
- return null;
- }
- }
-
- private void sendRequest(final StateMachine.Fsm fsm, RequestEvent request) {
- TSOProto.Request req = request.getRequest();
-
- if (req.hasTimestampRequest()) {
- timestampRequests.add(new RequestAndTimeout(request, newTimeout(new TimestampRequestTimeoutEvent())));
- } else if (req.hasCommitRequest()) {
- TSOProto.CommitRequest commitReq = req.getCommitRequest();
- commitRequests.put(commitReq.getStartTimestamp(), new RequestAndTimeout(
- request, newTimeout(new CommitRequestTimeoutEvent(commitReq.getStartTimestamp()))));
- } else {
- request.error(new IllegalArgumentException("Unknown request type"));
- return;
- }
- ChannelFuture f = channel.write(req);
-
- f.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture future) {
- if (!future.isSuccess()) {
- fsm.sendEvent(new ErrorEvent(future.getCause()));
- }
- }
- });
- }
-
- private void handleResponse(ResponseEvent response) {
- TSOProto.Response resp = response.getParam();
- if (resp.hasTimestampResponse()) {
- if (timestampRequests.size() == 0) {
- LOG.debug("Received timestamp response when no requests outstanding");
- return;
- }
- RequestAndTimeout e = timestampRequests.remove();
- e.getRequest().success(resp.getTimestampResponse().getStartTimestamp());
- if (e.getTimeout() != null) {
- e.getTimeout().cancel();
- }
- } else if (resp.hasCommitResponse()) {
- long startTimestamp = resp.getCommitResponse().getStartTimestamp();
- RequestAndTimeout e = commitRequests.remove(startTimestamp);
- if (e == null) {
- LOG.debug("Received commit response for request that doesn't exist. Start TS: {}", startTimestamp);
- return;
- }
- if (e.getTimeout() != null) {
- e.getTimeout().cancel();
- }
- if (resp.getCommitResponse().getAborted()) {
- e.getRequest().error(new AbortException());
- } else {
- // Check if the commit response received implies heuristic
- // actions during commit (because there's a new TSO master
- // replica) and inform the caller (e.g. the TxMgr) about it
- if (resp.getCommitResponse().getMakeHeuristicDecision()) {
- e.getRequest().error(new NewTSOException());
- } else {
- e.getRequest().success(resp.getCommitResponse().getCommitTimestamp());
- }
- }
- }
- }
-
- public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
- if (!timestampRequests.isEmpty()) {
- RequestAndTimeout r = timestampRequests.remove();
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- }
- return this;
- }
-
- public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
- long startTimestamp = e.getStartTimestamp();
- if (commitRequests.containsKey(startTimestamp)) {
- RequestAndTimeout r = commitRequests.remove(startTimestamp);
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- }
- return this;
- }
-
- public StateMachine.State handleEvent(CloseEvent e) {
- LOG.debug("CONNECTED STATE: CloseEvent");
- timeoutExecutor.stop();
- closeChannelAndErrorRequests();
- fsm.deferEvent(e);
- return new ClosingState(fsm);
- }
-
- public StateMachine.State handleEvent(RequestEvent e) {
- sendRequest(fsm, e);
- return this;
- }
-
- public StateMachine.State handleEvent(ResponseEvent e) {
- handleResponse(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- LOG.debug("CONNECTED STATE: ErrorEvent");
- timeoutExecutor.stop();
- handleError(fsm);
- return new ClosingState(fsm);
- }
-
- private void handleError(StateMachine.Fsm fsm) {
- LOG.debug("CONNECTED STATE: Cancelling Timeouts in handleError");
- while (timestampRequests.size() > 0) {
- RequestAndTimeout r = timestampRequests.remove();
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- }
- Iterator<Map.Entry<Long, RequestAndTimeout>> iter = commitRequests.entrySet().iterator();
- while (iter.hasNext()) {
- RequestAndTimeout r = iter.next().getValue();
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- queueRetryOrError(fsm, r.getRequest());
- iter.remove();
- }
- channel.close();
- }
-
- private void queueRetryOrError(StateMachine.Fsm fsm, RequestEvent e) {
- if (e.getRetriesLeft() > 0) {
- e.decrementRetries();
- if (e.getRequest().hasCommitRequest()) {
- TSOProto.CommitRequest commitRequest = e.getRequest().getCommitRequest();
- if (!commitRequest.getIsRetry()) { // Create a new retry for the commit request
- TSOProto.Request.Builder builder = TSOProto.Request.newBuilder();
- TSOProto.CommitRequest.Builder commitBuilder = TSOProto.CommitRequest.newBuilder();
- commitBuilder.mergeFrom(commitRequest);
- commitBuilder.setIsRetry(true);
- builder.setCommitRequest(commitBuilder.build());
- e.setRequest(builder.build());
- }
- }
- fsm.sendEvent(e);
- } else {
- e.error(
- new ServiceUnavailableException("Number of retries exceeded. This API request failed permanently"));
- }
- }
-
- private void closeChannelAndErrorRequests() {
- channel.close();
- for (RequestAndTimeout r : timestampRequests) {
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- r.getRequest().error(new ClosingException());
- }
- for (RequestAndTimeout r : commitRequests.values()) {
- if (r.getTimeout() != null) {
- r.getTimeout().cancel();
- }
- r.getRequest().error(new ClosingException());
- }
- }
- }
-
- private class ClosingState extends BaseState {
-
- ClosingState(StateMachine.Fsm fsm) {
- super(fsm);
- LOG.debug("NEW STATE: CLOSING");
- }
-
- public StateMachine.State handleEvent(TimestampRequestTimeoutEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(CommitRequestTimeoutEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(ErrorEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(ResponseEvent e) {
- // Ignored. They will be retried or errored
- return this;
- }
-
- public StateMachine.State handleEvent(UserEvent e) {
- fsm.deferEvent(e);
- return this;
- }
-
- public StateMachine.State handleEvent(ChannelClosedEvent e) {
- return new DisconnectedState(fsm);
- }
-
- public StateMachine.State handleEvent(HandshakeTimeoutEvent e) {
- return this;
- }
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper classes & methods
- // ----------------------------------------------------------------------------------------------------------------
-
- private class Handler extends SimpleChannelHandler {
-
- private StateMachine.Fsm fsm;
-
- Handler(StateMachine.Fsm fsm) {
- this.fsm = fsm;
- }
-
- @Override
- public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e) {
- currentChannel = e.getChannel();
- LOG.debug("HANDLER (CHANNEL CONNECTED): Connection {}. Sending connected event to FSM", e);
- fsm.sendEvent(new ConnectedEvent(e.getChannel()));
- }
-
- @Override
- public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- LOG.debug("HANDLER (CHANNEL DISCONNECTED): Connection {}. Sending error event to FSM", e);
- fsm.sendEvent(new ErrorEvent(new ConnectionException()));
- }
-
- @Override
- public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
- LOG.debug("HANDLER (CHANNEL CLOSED): Connection {}. Sending channel closed event to FSM", e);
- fsm.sendEvent(new ChannelClosedEvent(new ConnectionException()));
- }
-
- @Override
- public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) {
- if (e.getMessage() instanceof TSOProto.Response) {
- fsm.sendEvent(new ResponseEvent((TSOProto.Response) e.getMessage()));
- } else {
- LOG.warn("Received unknown message", e.getMessage());
- }
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
- LOG.error("Error on channel {}", ctx.getChannel(), e.getCause());
- fsm.sendEvent(new ErrorEvent(e.getCause()));
- }
- }
-
- private synchronized void setTSOAddress(String host, int port) {
- tsoAddr = new InetSocketAddress(host, port);
- }
-
- private synchronized InetSocketAddress getAddress() {
- return tsoAddr;
- }
-
- private void configureCurrentTSOServerZNodeCache(String currentTsoPath) {
- try {
- currentTSOZNode = new NodeCache(zkClient, currentTsoPath);
- currentTSOZNode.getListenable().addListener(this);
- currentTSOZNode.start(true);
- } catch (Exception e) {
- throw new IllegalStateException("Cannot start watcher on current TSO Server ZNode: " + e.getMessage());
- }
- }
-
- private String getCurrentTSOInfoFoundInZK(String currentTsoPath) {
- ChildData currentTSOData = currentTSOZNode.getCurrentData();
- if (currentTSOData == null) {
- throw new IllegalStateException("No data found in ZKNode " + currentTsoPath);
- }
- byte[] currentTSOAndEpochAsBytes = currentTSOData.getData();
- if (currentTSOAndEpochAsBytes == null) {
- throw new IllegalStateException("No data found for current TSO in ZKNode " + currentTsoPath);
- }
- return new String(currentTSOAndEpochAsBytes, Charsets.UTF_8);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
deleted file mode 100644
index 0999065..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOFuture.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import java.util.concurrent.Executor;
-import java.util.concurrent.Future;
-
-public interface TSOFuture<T> extends Future<T> {
- public void addListener(Runnable listener, Executor executor);
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java b/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
deleted file mode 100644
index 553a15c..0000000
--- a/transaction-client/src/main/java/com/yahoo/omid/tsoclient/TSOProtocol.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import java.util.Set;
-
-/**
- * Defines the protocol used on the client side to abstract communication to the TSO server
- */
-public interface TSOProtocol {
-
- /**
- * Returns a new timestamp assigned by on the server-side
- * @return the newly assigned timestamp as a future. If an error was detected, the future will contain a
- * corresponding protocol exception
- * @see TimestampOracle
- * @see TSOServer
- */
- TSOFuture<Long> getNewStartTimestamp();
-
- /**
- * Returns the result of the conflict detection made on the server-side for the specified transaction
- * @param transactionId
- * the transaction to check for conflicts
- * @param writeSet
- * the writeSet of the transaction, which includes all the modified cells
- * @return the commit timestamp as a future if the transaction was committed. If the transaction was aborted due
- * to conflicts with a concurrent transaction, the future will include an AbortException. If an error was detected,
- * the future will contain a corresponding protocol exception
- * @see TimestampOracle
- * @see TSOServer
- */
- TSOFuture<Long> commit(long transactionId, Set<? extends CellId> writeSet);
-
- /**
- * Closes the communication with the TSO server
- * @return nothing. If an error was detected, the future will contain a corresponding protocol exception
- */
- TSOFuture<Void> close();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/main/resources/omid-client-config.yml
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/resources/omid-client-config.yml b/transaction-client/src/main/resources/omid-client-config.yml
index 7e362bb..daed7d3 100644
--- a/transaction-client/src/main/resources/omid-client-config.yml
+++ b/transaction-client/src/main/resources/omid-client-config.yml
@@ -7,13 +7,13 @@
# ---------------------------------------------------------------------------------------------------------------------
# Direct connection to host:port
-connectionType: !!com.yahoo.omid.tsoclient.OmidClientConfiguration$ConnType DIRECT
+connectionType: !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType DIRECT
connectionString: "localhost:54758"
# When Omid is working in High Availability mode, two or more replicas of the TSO server are running in primary/backup
# mode. When a TSO server replica is elected as master, it publishes its address through ZK. In order to configure
# the Omid client to access the TSO server in HA mode:
-# 1) set 'connectionType' to !!com.yahoo.omid.tsoclient.OmidClientConfiguration$ConnType HA
+# 1) set 'connectionType' to !!com.yahoo.omid.tso.client.OmidClientConfiguration$ConnType HA
# 2) set 'connectionString' to the ZK cluster connection string where the server is publishing its address
zkConnectionTimeoutInSecs: 10
# In HA mode, make sure that the next settings match same settings on the TSO server side
@@ -36,4 +36,4 @@ executorThreads: 3
# Configure whether the TM performs the post-commit actions for a tx (update shadow cells and clean commit table entry)
# before returning to the control to the client (SYNC) or in parallel (ASYNC)
-postCommitMode: !!com.yahoo.omid.tsoclient.OmidClientConfiguration$PostCommitMode SYNC
\ No newline at end of file
+postCommitMode: !!com.yahoo.omid.tso.client.OmidClientConfiguration$PostCommitMode SYNC
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java b/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java
new file mode 100644
index 0000000..deca92f
--- /dev/null
+++ b/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestMockTSOClient.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.google.common.collect.Sets;
+import com.yahoo.omid.committable.CommitTable;
+import com.yahoo.omid.committable.InMemoryCommitTable;
+import com.yahoo.omid.tso.util.DummyCellIdImpl;
+import org.testng.Assert;
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.ExecutionException;
+
+import static org.testng.AssertJUnit.assertEquals;
+
+public class TestMockTSOClient {
+
+ final static public CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
+ final static public CellId c2 = new DummyCellIdImpl(-0xfeedcafeL);
+
+ @Test(timeOut = 10000)
+ public void testConflicts() throws Exception {
+ CommitTable commitTable = new InMemoryCommitTable();
+ TSOProtocol client = new MockTSOClient(commitTable.getWriter());
+
+ long tr1 = client.getNewStartTimestamp().get();
+ long tr2 = client.getNewStartTimestamp().get();
+
+ client.commit(tr1, Sets.newHashSet(c1)).get();
+
+ try {
+ client.commit(tr2, Sets.newHashSet(c1, c2)).get();
+ Assert.fail("Shouldn't have committed");
+ } catch (ExecutionException ee) {
+ assertEquals("Should have aborted", ee.getCause().getClass(), AbortException.class);
+ }
+ }
+
+ @Test(timeOut = 10000)
+ public void testWatermarkUpdate() throws Exception {
+ CommitTable commitTable = new InMemoryCommitTable();
+ TSOProtocol client = new MockTSOClient(commitTable.getWriter());
+ CommitTable.Client commitTableClient = commitTable.getClient();
+
+ long tr1 = client.getNewStartTimestamp().get();
+ client.commit(tr1, Sets.newHashSet(c1)).get();
+
+ long initWatermark = commitTableClient.readLowWatermark().get();
+
+ long tr2 = client.getNewStartTimestamp().get();
+ client.commit(tr2, Sets.newHashSet(c1)).get();
+
+ long newWatermark = commitTableClient.readLowWatermark().get();
+ AssertJUnit.assertTrue("new low watermark should be bigger", newWatermark > initWatermark);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java b/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java
new file mode 100644
index 0000000..731401e
--- /dev/null
+++ b/transaction-client/src/test/java/com/yahoo/omid/tso/client/TestOmidClientConfiguration.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TestOmidClientConfiguration {
+
+ @Test
+ public void testYamlReading() {
+ OmidClientConfiguration configuration = new OmidClientConfiguration();
+ Assert.assertNotNull(configuration.getConnectionString());
+ Assert.assertNotNull(configuration.getConnectionType());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java b/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
deleted file mode 100644
index 5ea5ffb..0000000
--- a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestMockTSOClient.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import com.google.common.collect.Sets;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.InMemoryCommitTable;
-import com.yahoo.omid.tso.util.DummyCellIdImpl;
-import org.testng.Assert;
-import org.testng.AssertJUnit;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.ExecutionException;
-
-import static org.testng.AssertJUnit.assertEquals;
-
-public class TestMockTSOClient {
-
- final static public CellId c1 = new DummyCellIdImpl(0xdeadbeefL);
- final static public CellId c2 = new DummyCellIdImpl(-0xfeedcafeL);
-
- @Test(timeOut = 10000)
- public void testConflicts() throws Exception {
- CommitTable commitTable = new InMemoryCommitTable();
- TSOProtocol client = new MockTSOClient(commitTable.getWriter());
-
- long tr1 = client.getNewStartTimestamp().get();
- long tr2 = client.getNewStartTimestamp().get();
-
- client.commit(tr1, Sets.newHashSet(c1)).get();
-
- try {
- client.commit(tr2, Sets.newHashSet(c1, c2)).get();
- Assert.fail("Shouldn't have committed");
- } catch (ExecutionException ee) {
- assertEquals("Should have aborted", ee.getCause().getClass(), AbortException.class);
- }
- }
-
- @Test(timeOut = 10000)
- public void testWatermarkUpdate() throws Exception {
- CommitTable commitTable = new InMemoryCommitTable();
- TSOProtocol client = new MockTSOClient(commitTable.getWriter());
- CommitTable.Client commitTableClient = commitTable.getClient();
-
- long tr1 = client.getNewStartTimestamp().get();
- client.commit(tr1, Sets.newHashSet(c1)).get();
-
- long initWatermark = commitTableClient.readLowWatermark().get();
-
- long tr2 = client.getNewStartTimestamp().get();
- client.commit(tr2, Sets.newHashSet(c1)).get();
-
- long newWatermark = commitTableClient.readLowWatermark().get();
- AssertJUnit.assertTrue("new low watermark should be bigger", newWatermark > initWatermark);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java b/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
deleted file mode 100644
index 497ec6d..0000000
--- a/transaction-client/src/test/java/com/yahoo/omid/tsoclient/TestOmidClientConfiguration.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tsoclient;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-public class TestOmidClientConfiguration {
-
- @Test
- public void testYamlReading() {
- OmidClientConfiguration configuration = new OmidClientConfiguration();
- Assert.assertNotNull(configuration.getConnectionString());
- Assert.assertNotNull(configuration.getConnectionType());
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java b/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
index 4da3c45..71e2ecb 100644
--- a/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/TestLeaseManager.java
@@ -33,7 +33,7 @@ import org.testng.annotations.Test;
import java.io.IOException;
-import static com.yahoo.omid.tsoclient.TSOClient.DEFAULT_ZK_CLUSTER;
+import static com.yahoo.omid.tso.client.TSOClient.DEFAULT_ZK_CLUSTER;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
import static org.mockito.Mockito.mock;
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java
new file mode 100644
index 0000000..eeff214
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientAccessor.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.yahoo.statemachine.StateMachine.FsmImpl;
+
+public class TSOClientAccessor {
+
+ public static void closeChannel(TSOClient tsoClient) throws InterruptedException {
+ FsmImpl fsm = (FsmImpl) tsoClient.fsm;
+ TSOClient.ConnectedState connectedState = (TSOClient.ConnectedState) fsm.getState();
+ connectedState.channel.close().await();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/c6410f7d/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java
----------------------------------------------------------------------
diff --git a/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java
new file mode 100644
index 0000000..03dbbd7
--- /dev/null
+++ b/tso-server/src/test/java/com/yahoo/omid/tso/client/TSOClientOneShot.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yahoo.omid.tso.client;
+
+import com.yahoo.omid.proto.TSOProto;
+import com.yahoo.omid.proto.TSOProto.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Communication endpoint for TSO clients.
+ */
+public class TSOClientOneShot {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TSOClientOneShot.class);
+
+ private final String host;
+ private final int port;
+
+ public TSOClientOneShot(String host, int port) {
+
+ this.host = host;
+ this.port = port;
+
+ }
+
+ public TSOProto.Response makeRequest(TSOProto.Request request)
+ throws InterruptedException, ExecutionException {
+ TSOClientRaw raw = new TSOClientRaw(host, port);
+
+ // do handshake
+ TSOProto.HandshakeRequest.Builder handshake = TSOProto.HandshakeRequest.newBuilder();
+ handshake.setClientCapabilities(TSOProto.Capabilities.newBuilder().build());
+ raw.write(TSOProto.Request.newBuilder()
+ .setHandshakeRequest(handshake.build()).build());
+ Response response = raw.getResponse().get();
+ assert (response.getHandshakeResponse().getClientCompatible());
+
+ raw.write(request);
+ response = raw.getResponse().get();
+
+ raw.close();
+ return response;
+ }
+
+}
[29/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompaction.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompaction.java b/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompaction.java
new file mode 100644
index 0000000..939b81f
--- /dev/null
+++ b/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompaction.java
@@ -0,0 +1,1299 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.omid.HBaseShims;
+import org.apache.omid.TestUtils;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
+import org.apache.omid.tso.TSOServer;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
+import org.apache.hadoop.hbase.client.Row;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.coprocessor.AggregationClient;
+import org.apache.hadoop.hbase.client.coprocessor.LongColumnInterpreter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+public class TestCompaction {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class);
+
+ private static final String TEST_FAMILY = "test-fam";
+ private static final String TEST_QUALIFIER = "test-qual";
+
+ private final byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ private final byte[] qual = Bytes.toBytes(TEST_QUALIFIER);
+ private final byte[] data = Bytes.toBytes("testWrite-1");
+
+ private static final int MAX_VERSIONS = 3;
+
+ private Random randomGenerator;
+ private AbstractTransactionManager tm;
+
+ private Injector injector;
+
+ private HBaseAdmin admin;
+ private Configuration hbaseConf;
+ private HBaseTestingUtility hbaseTestUtil;
+ private MiniHBaseCluster hbaseCluster;
+
+ private TSOServer tso;
+
+ private AggregationClient aggregationClient;
+ private CommitTable commitTable;
+//<<<<<<< HEAD
+// private TSOClient client;
+ private PostCommitActions syncPostCommitter;
+//=======
+//>>>>>>> open-source-master
+
+ @BeforeClass
+ public void setupTestCompation() throws Exception {
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setPort(1234);
+ tsoConfig.setMaxItems(1);
+ injector = Guice.createInjector(new TSOForHBaseCompactorTestModule(tsoConfig));
+ hbaseConf = injector.getInstance(Configuration.class);
+ HBaseCommitTableConfig hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);
+ HBaseTimestampStorageConfig hBaseTimestampStorageConfig = injector.getInstance(HBaseTimestampStorageConfig.class);
+
+ // settings required for #testDuplicateDeletes()
+ hbaseConf.setInt("hbase.hstore.compaction.min", 2);
+ hbaseConf.setInt("hbase.hstore.compaction.max", 2);
+
+ setupHBase();
+ aggregationClient = new AggregationClient(hbaseConf);
+ admin = new HBaseAdmin(hbaseConf);
+ createRequiredHBaseTables(hBaseTimestampStorageConfig, hBaseCommitTableConfig);
+ setupTSO();
+
+ commitTable = injector.getInstance(CommitTable.class);
+
+//<<<<<<< HEAD
+// client = TSOClient.newBuilder().withConfiguration(clientConf).build();
+//
+//=======
+//>>>>>>> open-source-master
+ }
+
+ private void setupHBase() throws Exception {
+ LOG.info("********** Setting up HBase **********");
+ hbaseTestUtil = new HBaseTestingUtility(hbaseConf);
+ LOG.info("********** Creating HBase MiniCluster **********");
+ hbaseCluster = hbaseTestUtil.startMiniCluster(1);
+ }
+
+ private void createRequiredHBaseTables(HBaseTimestampStorageConfig timestampStorageConfig,
+ HBaseCommitTableConfig hBaseCommitTableConfig) throws IOException {
+ createTableIfNotExists(timestampStorageConfig.getTableName(), timestampStorageConfig.getFamilyName().getBytes());
+
+ createTableIfNotExists(hBaseCommitTableConfig.getTableName(), hBaseCommitTableConfig.getCommitTableFamily(), hBaseCommitTableConfig.getLowWatermarkFamily());
+ }
+
+ private void createTableIfNotExists(String tableName, byte[]... families) throws IOException {
+ if (!admin.tableExists(tableName)) {
+ LOG.info("Creating {} table...", tableName);
+ HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
+
+ for (byte[] family : families) {
+ HColumnDescriptor datafam = new HColumnDescriptor(family);
+ datafam.setMaxVersions(MAX_VERSIONS);
+ desc.addFamily(datafam);
+ }
+
+ desc.addCoprocessor("org.apache.hadoop.hbase.coprocessor.AggregateImplementation");
+ admin.createTable(desc);
+ for (byte[] family : families) {
+ CompactorUtil.enableOmidCompaction(hbaseConf, TableName.valueOf(tableName), family);
+ }
+ }
+
+ }
+
+ private void setupTSO() throws IOException, InterruptedException {
+ tso = injector.getInstance(TSOServer.class);
+ tso.startAndWait();
+ TestUtils.waitForSocketListening("localhost", 1234, 100);
+ Thread.currentThread().setName("UnitTest(s) thread");
+ }
+
+ @AfterClass
+ public void cleanupTestCompation() throws Exception {
+ teardownTSO();
+ hbaseCluster.shutdown();
+ }
+
+ private void teardownTSO() throws IOException, InterruptedException {
+ tso.stopAndWait();
+ TestUtils.waitForSocketNotListening("localhost", 1234, 1000);
+ }
+
+ @BeforeMethod
+ public void setupTestCompactionIndividualTest() throws Exception {
+ randomGenerator = new Random(0xfeedcafeL);
+ tm = spy((AbstractTransactionManager) newTransactionManager());
+ }
+
+ private TransactionManager newTransactionManager() throws Exception {
+//<<<<<<< HEAD
+// CommitTable.Client commitTableClient = commitTable.getClient();
+// syncPostCommitter =
+// spy(new HBaseSyncPostCommitter(new NullMetricsProvider(),commitTableClient));
+//
+// return HBaseTransactionManager.newBuilder()
+// .withConfiguration(hbaseConf)
+// .withCommitTableClient(commitTableClient)
+// .withTSOClient(client)
+// .postCommitter(syncPostCommitter)
+//=======
+ HBaseOmidClientConfiguration hbaseOmidClientConf = new HBaseOmidClientConfiguration();
+ hbaseOmidClientConf.setConnectionString("localhost:1234");
+ hbaseOmidClientConf.setHBaseConfiguration(hbaseConf);
+ CommitTable.Client commitTableClient = commitTable.getClient();
+ syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(),commitTableClient));
+ return HBaseTransactionManager.builder(hbaseOmidClientConf)
+ .postCommitter(syncPostCommitter)
+ .commitTableClient(commitTableClient)
+//>>>>>>> open-source-master
+ .build();
+ }
+
+ @Test
+ public void testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction() throws Throwable {
+ String TEST_TABLE = "testStandardTXsWithShadowCellsAndWithSTBelowAndAboveLWMArePresevedAfterCompaction";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ final int ROWS_TO_ADD = 5;
+
+ long fakeAssignedLowWatermark = 0L;
+ for (int i = 0; i < ROWS_TO_ADD; ++i) {
+ long rowId = randomGenerator.nextLong();
+ Transaction tx = tm.begin();
+ if (i == (ROWS_TO_ADD / 2)) {
+ fakeAssignedLowWatermark = tx.getTransactionId();
+ LOG.info("AssignedLowWatermark " + fakeAssignedLowWatermark);
+ }
+ Put put = new Put(Bytes.toBytes(rowId));
+ put.add(fam, qual, data);
+ txTable.put(tx, put);
+ tm.commit(tx);
+ }
+
+ LOG.info("Flushing table {}", TEST_TABLE);
+ admin.flush(TEST_TABLE);
+
+ // Return a LWM that triggers compaction & stays between 1 and the max start timestamp assigned to previous TXs
+ LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
+ OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
+ .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ CommitTable.Client commitTableClient = spy(commitTable.getClient());
+ SettableFuture<Long> f = SettableFuture.create();
+ f.set(fakeAssignedLowWatermark);
+ doReturn(f).when(commitTableClient).readLowWatermark();
+ omidCompactor.commitTableClientQueue.add(commitTableClient);
+ LOG.info("Compacting table {}", TEST_TABLE);
+ admin.majorCompact(TEST_TABLE);
+
+ LOG.info("Sleeping for 3 secs");
+ Thread.sleep(3000);
+ LOG.info("Waking up after 3 secs");
+
+ // No rows should have been discarded after compacting
+ assertEquals("Rows in table after compacting should be " + ROWS_TO_ADD, ROWS_TO_ADD, rowCount(TEST_TABLE, fam));
+ }
+
+ @Test
+ public void testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction() throws Exception {
+ String TEST_TABLE = "testTXWithoutShadowCellsAndWithSTBelowLWMGetsShadowCellHealedAfterCompaction";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // The following line emulates a crash after commit that is observed in (*) below
+ doThrow(new RuntimeException()).when(syncPostCommitter).updateShadowCells(any(HBaseTransaction.class));
+
+ HBaseTransaction problematicTx = (HBaseTransaction) tm.begin();
+
+ long row = randomGenerator.nextLong();
+
+ // Test shadow cell are created properly
+ Put put = new Put(Bytes.toBytes(row));
+ put.add(fam, qual, data);
+ txTable.put(problematicTx, put);
+ try {
+ tm.commit(problematicTx);
+ } catch (Exception e) { // (*) Crash
+ // Do nothing
+ }
+
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(row),
+ fam,
+ qual,
+ problematicTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(row),
+ fam,
+ qual,
+ problematicTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+
+ // Return a LWM that triggers compaction and has all the possible start timestamps below it
+ LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
+ OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
+ .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ CommitTable.Client commitTableClient = spy(commitTable.getClient());
+ SettableFuture<Long> f = SettableFuture.create();
+ f.set(Long.MAX_VALUE);
+ doReturn(f).when(commitTableClient).readLowWatermark();
+ omidCompactor.commitTableClientQueue.add(commitTableClient);
+
+ LOG.info("Flushing table {}", TEST_TABLE);
+ admin.flush(TEST_TABLE);
+
+ LOG.info("Compacting table {}", TEST_TABLE);
+ admin.majorCompact(TEST_TABLE);
+
+ LOG.info("Sleeping for 3 secs");
+ Thread.sleep(3000);
+ LOG.info("Waking up after 3 secs");
+
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(row),
+ fam,
+ qual,
+ problematicTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertTrue("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(row),
+ fam,
+ qual,
+ problematicTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ }
+
+ @Test
+ public void testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction()
+ throws Throwable {
+ String
+ TEST_TABLE =
+ "testNeverendingTXsWithSTBelowAndAboveLWMAreDiscardedAndPreservedRespectivelyAfterCompaction";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // The KV in this transaction should be discarded
+ HBaseTransaction neverendingTxBelowLowWatermark = (HBaseTransaction) tm.begin();
+ long rowId = randomGenerator.nextLong();
+ Put put = new Put(Bytes.toBytes(rowId));
+ put.add(fam, qual, data);
+ txTable.put(neverendingTxBelowLowWatermark, put);
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTxBelowLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTxBelowLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+
+ // The KV in this transaction should be added without the shadow cells
+ HBaseTransaction neverendingTxAboveLowWatermark = (HBaseTransaction) tm.begin();
+ long anotherRowId = randomGenerator.nextLong();
+ put = new Put(Bytes.toBytes(anotherRowId));
+ put.add(fam, qual, data);
+ txTable.put(neverendingTxAboveLowWatermark, put);
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(anotherRowId),
+ fam,
+ qual,
+ neverendingTxAboveLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId),
+ fam,
+ qual,
+ neverendingTxAboveLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+
+ assertEquals("Rows in table before flushing should be 2", 2, rowCount(TEST_TABLE, fam));
+ LOG.info("Flushing table {}", TEST_TABLE);
+ admin.flush(TEST_TABLE);
+ assertEquals("Rows in table after flushing should be 2", 2, rowCount(TEST_TABLE, fam));
+
+ // Return a LWM that triggers compaction and stays between both ST of TXs, so assign 1st TX's start timestamp
+ LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
+ OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
+ .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ CommitTable.Client commitTableClient = spy(commitTable.getClient());
+ SettableFuture<Long> f = SettableFuture.create();
+ f.set(neverendingTxBelowLowWatermark.getStartTimestamp());
+ doReturn(f).when(commitTableClient).readLowWatermark();
+ omidCompactor.commitTableClientQueue.add(commitTableClient);
+ LOG.info("Compacting table {}", TEST_TABLE);
+ admin.majorCompact(TEST_TABLE);
+
+ LOG.info("Sleeping for 3 secs");
+ Thread.sleep(3000);
+ LOG.info("Waking up after 3 secs");
+
+ // One row should have been discarded after compacting
+ assertEquals("There should be only one row in table after compacting", 1, rowCount(TEST_TABLE, fam));
+ // The row from the TX below the LWM should not be there (nor its Shadow Cell)
+ assertFalse("Cell should not be there",
+ CellUtils.hasCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTxBelowLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTxBelowLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ // The row from the TX above the LWM should be there without the Shadow Cell
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(anotherRowId),
+ fam,
+ qual,
+ neverendingTxAboveLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(anotherRowId),
+ fam,
+ qual,
+ neverendingTxAboveLowWatermark.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+
+ }
+
+ @Test
+ public void testRowsUnalteredWhenCommitTableCannotBeReached() throws Throwable {
+ String TEST_TABLE = "testRowsUnalteredWhenCommitTableCannotBeReached";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // The KV in this transaction should be discarded but in the end should remain there because
+ // the commit table won't be accessed (simulating an error on access)
+ HBaseTransaction neverendingTx = (HBaseTransaction) tm.begin();
+ long rowId = randomGenerator.nextLong();
+ Put put = new Put(Bytes.toBytes(rowId));
+ put.add(fam, qual, data);
+ txTable.put(neverendingTx, put);
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+
+ assertEquals("There should be only one rows in table before flushing", 1, rowCount(TEST_TABLE, fam));
+ LOG.info("Flushing table {}", TEST_TABLE);
+ admin.flush(TEST_TABLE);
+ assertEquals("There should be only one rows in table after flushing", 1, rowCount(TEST_TABLE, fam));
+
+ // Break access to CommitTable functionality in Compactor
+ LOG.info("Regions in table {}: {}", TEST_TABLE, hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).size());
+ OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(TEST_TABLE)).get(0)
+ .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ CommitTable.Client commitTableClient = spy(commitTable.getClient());
+ SettableFuture<Long> f = SettableFuture.create();
+ f.setException(new IOException("Unable to read"));
+ doReturn(f).when(commitTableClient).readLowWatermark();
+ omidCompactor.commitTableClientQueue.add(commitTableClient);
+
+ LOG.info("Compacting table {}", TEST_TABLE);
+ admin.majorCompact(TEST_TABLE); // Should trigger the error when accessing CommitTable funct.
+
+ LOG.info("Sleeping for 3 secs");
+ Thread.sleep(3000);
+ LOG.info("Waking up after 3 secs");
+
+ // All rows should be there after the failed compaction
+ assertEquals("There should be only one row in table after compacting", 1, rowCount(TEST_TABLE, fam));
+ assertTrue("Cell should be there",
+ CellUtils.hasCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ assertFalse("Shadow cell should not be there",
+ CellUtils.hasShadowCell(Bytes.toBytes(rowId),
+ fam,
+ qual,
+ neverendingTx.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ }
+
+ @Test
+ public void testOriginalTableParametersAreAvoidedAlsoWhenCompacting() throws Throwable {
+ String TEST_TABLE = "testOriginalTableParametersAreAvoidedAlsoWhenCompacting";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ long rowId = randomGenerator.nextLong();
+ for (int versionCount = 0; versionCount <= (2 * MAX_VERSIONS); versionCount++) {
+ Transaction tx = tm.begin();
+ Put put = new Put(Bytes.toBytes(rowId));
+ put.add(fam, qual, Bytes.toBytes("testWrite-" + versionCount));
+ txTable.put(tx, put);
+ tm.commit(tx);
+ }
+
+ Transaction tx = tm.begin();
+ Get get = new Get(Bytes.toBytes(rowId));
+ get.setMaxVersions(2 * MAX_VERSIONS);
+ assertEquals("Max versions should be set to " + (2 * MAX_VERSIONS), (2 * MAX_VERSIONS), get.getMaxVersions());
+ get.addColumn(fam, qual);
+ Result result = txTable.get(tx, get);
+ tm.commit(tx);
+ List<Cell> column = result.getColumnCells(fam, qual);
+ assertEquals("There should be only one version in the result", 1, column.size());
+
+ assertEquals("There should be only one row in table before flushing", 1, rowCount(TEST_TABLE, fam));
+ LOG.info("Flushing table {}", TEST_TABLE);
+ admin.flush(TEST_TABLE);
+ assertEquals("There should be only one row in table after flushing", 1, rowCount(TEST_TABLE, fam));
+
+ // Return a LWM that triggers compaction
+ compactEverything(TEST_TABLE);
+
+ // One row should have been discarded after compacting
+ assertEquals("There should be only one row in table after compacting", 1, rowCount(TEST_TABLE, fam));
+
+ tx = tm.begin();
+ get = new Get(Bytes.toBytes(rowId));
+ get.setMaxVersions(2 * MAX_VERSIONS);
+ assertEquals("Max versions should be set to " + (2 * MAX_VERSIONS), (2 * MAX_VERSIONS), get.getMaxVersions());
+ get.addColumn(fam, qual);
+ result = txTable.get(tx, get);
+ tm.commit(tx);
+ column = result.getColumnCells(fam, qual);
+ assertEquals("There should be only one version in the result", 1, column.size());
+ assertEquals("Values don't match",
+ "testWrite-" + (2 * MAX_VERSIONS),
+ Bytes.toString(CellUtil.cloneValue(column.get(0))));
+ }
+
+ // manually flush the regions on the region server.
+ // flushing like this prevents compaction running
+ // directly after the flush, which we want to avoid.
+ private void manualFlush(String tableName) throws Throwable {
+ LOG.info("Manually flushing all regions and waiting 2 secs");
+ HBaseShims.flushAllOnlineRegions(hbaseTestUtil.getHBaseCluster().getRegionServer(0),
+ TableName.valueOf(tableName));
+ TimeUnit.SECONDS.sleep(2);
+ }
+
+ @Test
+ public void testOldCellsAreDiscardedAfterCompaction() throws Exception {
+ String TEST_TABLE = "testOldCellsAreDiscardedAfterCompaction";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ byte[] rowId = Bytes.toBytes("row");
+
+ // Create 3 transactions modifying the same cell in a particular row
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put1 = new Put(rowId);
+ put1.add(fam, qual, Bytes.toBytes("testValue 1"));
+ txTable.put(tx1, put1);
+ tm.commit(tx1);
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Put put2 = new Put(rowId);
+ put2.add(fam, qual, Bytes.toBytes("testValue 2"));
+ txTable.put(tx2, put2);
+ tm.commit(tx2);
+
+ HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
+ Put put3 = new Put(rowId);
+ put3.add(fam, qual, Bytes.toBytes("testValue 3"));
+ txTable.put(tx3, put3);
+ tm.commit(tx3);
+
+ // Before compaction, the three timestamped values for the cell should be there
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertTrue("Put cell of Tx1 should be there",
+ CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell of Tx1 should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
+ assertTrue("Put cell of Tx2 cell should be there",
+ CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell of Tx2 should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
+ assertTrue("Put cell of Tx3 cell should be there",
+ CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell of Tx3 should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
+
+ // Compact
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ // After compaction, only the last value for the cell should have survived
+ assertFalse("Put cell of Tx1 should not be there",
+ CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
+ assertFalse("Put shadow cell of Tx1 should not be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
+ assertFalse("Put cell of Tx2 should not be there",
+ CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
+ assertFalse("Put shadow cell of Tx2 should not be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
+ assertTrue("Put cell of Tx3 cell should be there",
+ CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell of Tx3 should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
+
+ // A new transaction after compaction should read the last value written
+ HBaseTransaction newTx1 = (HBaseTransaction) tm.begin();
+ Get newGet1 = new Get(rowId);
+ newGet1.addColumn(fam, qual);
+ Result result = txTable.get(newTx1, newGet1);
+ assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
+ // Write a new value
+ Put newPut1 = new Put(rowId);
+ newPut1.add(fam, qual, Bytes.toBytes("new testValue 1"));
+ txTable.put(newTx1, newPut1);
+
+ // Start a second new transaction
+ HBaseTransaction newTx2 = (HBaseTransaction) tm.begin();
+ // Commit first of the new tx
+ tm.commit(newTx1);
+
+ // The second transaction should still read the previous value
+ Get newGet2 = new Get(rowId);
+ newGet2.addColumn(fam, qual);
+ result = txTable.get(newTx2, newGet2);
+ assertEquals(Bytes.toBytes("testValue 3"), result.getValue(fam, qual));
+ tm.commit(newTx2);
+
+ // Only two values -the new written by newTx1 and the last value
+ // for the cell after compaction- should have survived
+ assertFalse("Put cell of Tx1 should not be there",
+ CellUtils.hasCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
+ assertFalse("Put shadow cell of Tx1 should not be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx1.getStartTimestamp(), getter));
+ assertFalse("Put cell of Tx2 should not be there",
+ CellUtils.hasCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
+ assertFalse("Put shadow cell of Tx2 should not be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx2.getStartTimestamp(), getter));
+ assertTrue("Put cell of Tx3 cell should be there",
+ CellUtils.hasCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell of Tx3 should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, tx3.getStartTimestamp(), getter));
+ assertTrue("Put cell of NewTx1 cell should be there",
+ CellUtils.hasCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell of NewTx1 should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual, newTx1.getStartTimestamp(), getter));
+ }
+
+ /**
+ * Tests a case where a temporary failure to flush causes the compactor to crash
+ */
+ @Test
+ public void testDuplicateDeletes() throws Throwable {
+ String TEST_TABLE = "testDuplicateDeletes";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // jump through hoops to trigger a minor compaction.
+ // a minor compaction will only run if there are enough
+ // files to be compacted, but that is less than the number
+ // of total files, in which case it will run a major
+ // compaction. The issue this is testing only shows up
+ // with minor compaction, as only Deletes can be duplicate
+ // and major compactions filter them out.
+ byte[] firstRow = "FirstRow".getBytes();
+ HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
+ Put put0 = new Put(firstRow);
+ put0.add(fam, qual, Bytes.toBytes("testWrite-1"));
+ txTable.put(tx0, put0);
+ tm.commit(tx0);
+
+ // create the first hfile
+ manualFlush(TEST_TABLE);
+
+ // write a row, it won't be committed
+ byte[] rowToBeCompactedAway = "compactMe".getBytes();
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put put1 = new Put(rowToBeCompactedAway);
+ put1.add(fam, qual, Bytes.toBytes("testWrite-1"));
+ txTable.put(tx1, put1);
+ txTable.flushCommits();
+
+ // write a row to trigger the double delete problem
+ byte[] row = "iCauseErrors".getBytes();
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Put put2 = new Put(row);
+ put2.add(fam, qual, Bytes.toBytes("testWrite-1"));
+ txTable.put(tx2, put2);
+ tm.commit(tx2);
+
+ HBaseTransaction tx3 = (HBaseTransaction) tm.begin();
+ Put put3 = new Put(row);
+ put3.add(fam, qual, Bytes.toBytes("testWrite-1"));
+ txTable.put(tx3, put3);
+ txTable.flushCommits();
+
+ // cause a failure on HBaseTM#preCommit();
+ Set<HBaseCellId> writeSet = tx3.getWriteSet();
+ assertEquals(1, writeSet.size());
+ List<HBaseCellId> newWriteSet = new ArrayList<>();
+ final AtomicBoolean flushFailing = new AtomicBoolean(true);
+ for (HBaseCellId id : writeSet) {
+ HTableInterface failableHTable = spy(id.getTable());
+ doAnswer(new Answer<Void>() {
+ @Override
+ public Void answer(InvocationOnMock invocation)
+ throws Throwable {
+ if (flushFailing.get()) {
+ throw new RetriesExhaustedWithDetailsException(new ArrayList<Throwable>(),
+ new ArrayList<Row>(), new ArrayList<String>());
+ } else {
+ invocation.callRealMethod();
+ }
+ return null;
+ }
+ }).when(failableHTable).flushCommits();
+
+ newWriteSet.add(new HBaseCellId(failableHTable,
+ id.getRow(), id.getFamily(),
+ id.getQualifier(), id.getTimestamp()));
+ }
+ writeSet.clear();
+ writeSet.addAll(newWriteSet);
+
+ try {
+ tm.commit(tx3);
+ fail("Shouldn't succeed");
+ } catch (TransactionException tme) {
+ flushFailing.set(false);
+ tm.rollback(tx3);
+ }
+
+ // create second hfile,
+ // it should contain multiple deletes
+ manualFlush(TEST_TABLE);
+
+ // create loads of files
+ byte[] anotherRow = "someotherrow".getBytes();
+ HBaseTransaction tx4 = (HBaseTransaction) tm.begin();
+ Put put4 = new Put(anotherRow);
+ put4.add(fam, qual, Bytes.toBytes("testWrite-1"));
+ txTable.put(tx4, put4);
+ tm.commit(tx4);
+
+ // create third hfile
+ manualFlush(TEST_TABLE);
+
+ // trigger minor compaction and give it time to run
+ setCompactorLWM(tx4.getStartTimestamp(), TEST_TABLE);
+ admin.compact(TEST_TABLE);
+ Thread.sleep(3000);
+
+ // check if the cell that should be compacted, is compacted
+ assertFalse("Cell should not be be there",
+ CellUtils.hasCell(rowToBeCompactedAway, fam, qual,
+ tx1.getStartTimestamp(),
+ new TTableCellGetterAdapter(txTable)));
+ }
+
+ @Test(timeOut = 60000)
+ public void testNonOmidCFIsUntouched() throws Throwable {
+ String TEST_TABLE = "testNonOmidCFIsUntouched";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ admin.disableTable(TEST_TABLE);
+ byte[] nonOmidCF = Bytes.toBytes("nonOmidCF");
+ byte[] nonOmidQual = Bytes.toBytes("nonOmidCol");
+ HColumnDescriptor nonomidfam = new HColumnDescriptor(nonOmidCF);
+ nonomidfam.setMaxVersions(MAX_VERSIONS);
+ admin.addColumn(TEST_TABLE, nonomidfam);
+ admin.enableTable(TEST_TABLE);
+
+ byte[] rowId = Bytes.toBytes("testRow");
+ Transaction tx = tm.begin();
+ Put put = new Put(rowId);
+ put.add(fam, qual, Bytes.toBytes("testValue"));
+ txTable.put(tx, put);
+
+ Put nonTxPut = new Put(rowId);
+ nonTxPut.add(nonOmidCF, nonOmidQual, Bytes.toBytes("nonTxVal"));
+ txTable.getHTable().put(nonTxPut);
+ txTable.flushCommits(); // to make sure it left the client
+
+ Get g = new Get(rowId);
+ Result result = txTable.getHTable().get(g);
+ assertEquals("Should be there, precompact",
+ 1, result.getColumnCells(nonOmidCF, nonOmidQual).size());
+ assertEquals("Should be there, precompact",
+ 1, result.getColumnCells(fam, qual).size());
+
+ compactEverything(TEST_TABLE);
+
+ result = txTable.getHTable().get(g);
+ assertEquals("Should be there, postcompact",
+ 1, result.getColumnCells(nonOmidCF, nonOmidQual).size());
+ assertEquals("Should not be there, postcompact",
+ 0, result.getColumnCells(fam, qual).size());
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Tests on tombstones and non-transactional Deletes
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Test that when a major compaction runs, cells that were deleted non-transactionally dissapear
+ */
+ @Test(timeOut = 60_000)
+ public void testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs() throws Throwable {
+ String TEST_TABLE = "testACellDeletedNonTransactionallyDoesNotAppearWhenAMajorCompactionOccurs";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HTable table = new HTable(hbaseConf, TEST_TABLE);
+
+ // Write first a value transactionally
+ HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("row1");
+ Put p0 = new Put(rowId);
+ p0.add(fam, qual, Bytes.toBytes("testValue-0"));
+ txTable.put(tx0, p0);
+ tm.commit(tx0);
+
+ // Then perform a non-transactional Delete
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ table.delete(d);
+
+ // Trigger a major compaction
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ // Then perform a non-tx (raw) scan...
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ ResultScanner scannerResults = table.getScanner(scan);
+
+ // ...and test the deleted cell is not there anymore
+ assertNull("There should be no results in scan results", scannerResults.next());
+
+ table.close();
+
+ }
+
+ /**
+ * Test that when a minor compaction runs, cells that were deleted non-transactionally are preserved. This is to
+ * allow users still access the cells when doing "improper" operations on a transactional table
+ */
+ @Test(timeOut = 60_000)
+ public void testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs() throws Throwable {
+ String TEST_TABLE = "testACellDeletedNonTransactionallyIsPreservedWhenMinorCompactionOccurs";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HTable table = new HTable(hbaseConf, TEST_TABLE);
+
+ // Configure the environment to create a minor compaction
+
+ // Write first a value transactionally
+ HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("row1");
+ Put p0 = new Put(rowId);
+ p0.add(fam, qual, Bytes.toBytes("testValue-0"));
+ txTable.put(tx0, p0);
+ tm.commit(tx0);
+
+ // create the first hfile
+ manualFlush(TEST_TABLE);
+
+ // Write another value transactionally
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put p1 = new Put(rowId);
+ p1.add(fam, qual, Bytes.toBytes("testValue-1"));
+ txTable.put(tx1, p1);
+ tm.commit(tx1);
+
+ // create the second hfile
+ manualFlush(TEST_TABLE);
+
+ // Write yet another value transactionally
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Put p2 = new Put(rowId);
+ p2.add(fam, qual, Bytes.toBytes("testValue-2"));
+ txTable.put(tx2, p2);
+ tm.commit(tx2);
+
+ // create a third hfile
+ manualFlush(TEST_TABLE);
+
+ // Then perform a non-transactional Delete
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ table.delete(d);
+
+ // create the fourth hfile
+ manualFlush(TEST_TABLE);
+
+ // Trigger the minor compaction
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+ admin.compact(TEST_TABLE);
+ Thread.sleep(5000);
+
+ // Then perform a non-tx (raw) scan...
+ Scan scan = new Scan();
+ scan.setRaw(true);
+ ResultScanner scannerResults = table.getScanner(scan);
+
+ // ...and test the deleted cell is still there
+ int count = 0;
+ Result scanResult;
+ List<Cell> listOfCellsScanned = new ArrayList<>();
+ while ((scanResult = scannerResults.next()) != null) {
+ listOfCellsScanned = scanResult.listCells(); // equivalent to rawCells()
+ count++;
+ }
+ assertEquals("There should be only one result in scan results", 1, count);
+ assertEquals("There should be three cell entries in the scan results (2 puts, 1 delete)", 3,
+ listOfCellsScanned.size());
+ boolean wasDeletedCellFound = false;
+ int numberOfDeletedCellsFound = 0;
+ for (Cell cell : listOfCellsScanned) {
+ if (CellUtil.isDelete(cell)) {
+ wasDeletedCellFound = true;
+ numberOfDeletedCellsFound++;
+ }
+ }
+ assertTrue("We should have found a non-transactionally deleted cell", wasDeletedCellFound);
+ assertEquals("There should be only only one deleted cell", 1, numberOfDeletedCellsFound);
+
+ table.close();
+ }
+
+ /**
+ * Test that when a minor compaction runs, tombstones are not cleaned up
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs() throws Throwable {
+ String TEST_TABLE = "testTombstonesAreNotCleanedUpWhenMinorCompactionOccurs";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // Configure the environment to create a minor compaction
+
+ HBaseTransaction tx0 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("case1");
+ Put p = new Put(rowId);
+ p.add(fam, qual, Bytes.toBytes("testValue-0"));
+ txTable.put(tx0, p);
+ tm.commit(tx0);
+
+ // create the first hfile
+ manualFlush(TEST_TABLE);
+
+ // Create the tombstone
+ HBaseTransaction deleteTx = (HBaseTransaction) tm.begin();
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(deleteTx, d);
+ tm.commit(deleteTx);
+
+ // create the second hfile
+ manualFlush(TEST_TABLE);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Put p1 = new Put(rowId);
+ p1.add(fam, qual, Bytes.toBytes("testValue-11"));
+ txTable.put(tx1, p1);
+ tm.commit(tx1);
+
+ // create the third hfile
+ manualFlush(TEST_TABLE);
+
+ HBaseTransaction lastTx = (HBaseTransaction) tm.begin();
+ Put p2 = new Put(rowId);
+ p2.add(fam, qual, Bytes.toBytes("testValue-222"));
+ txTable.put(lastTx, p2);
+ tm.commit(lastTx);
+
+ // Trigger the minor compaction
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+ admin.compact(TEST_TABLE);
+ Thread.sleep(5000);
+
+ // Checks on results after compaction
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertFalse("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx0.getStartTimestamp(), getter));
+ assertFalse("Put shadow cell should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx0.getStartTimestamp(), getter));
+ assertTrue("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Delete cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ deleteTx.getStartTimestamp(), getter));
+ assertTrue("Delete shadow cell should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ deleteTx.getStartTimestamp(), getter));
+ assertTrue("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ lastTx.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ lastTx.getStartTimestamp(), getter));
+ }
+
+
+ /**
+ * Test that when compaction runs, tombstones are cleaned up case1: 1 put (ts < lwm) then tombstone (ts > lwm)
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreCleanedUpCase1() throws Exception {
+ String TEST_TABLE = "testTombstonesAreCleanedUpCase1";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("case1");
+ Put p = new Put(rowId);
+ p.add(fam, qual, Bytes.toBytes("testValue"));
+ txTable.put(tx1, p);
+ tm.commit(tx1);
+
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ setCompactorLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(tx2, d);
+ tm.commit(tx2);
+
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertTrue("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Delete cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ assertTrue("Delete shadow cell should be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ }
+
+ /**
+ * Test that when compaction runs, tombstones are cleaned up case2: 1 put (ts < lwm) then tombstone (ts < lwm)
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreCleanedUpCase2() throws Exception {
+ String TEST_TABLE = "testTombstonesAreCleanedUpCase2";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("case2");
+ Put p = new Put(rowId);
+ p.add(fam, qual, Bytes.toBytes("testValue"));
+ txTable.put(tx1, p);
+ tm.commit(tx1);
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(tx2, d);
+ tm.commit(tx2);
+
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertFalse("Put cell shouldn't be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertFalse("Put shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertFalse("Delete cell shouldn't be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ assertFalse("Delete shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ }
+
+ /**
+ * Test that when compaction runs, tombstones are cleaned up case3: 1 put (ts < lwm) then tombstone (ts < lwm) not
+ * committed
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreCleanedUpCase3() throws Exception {
+ String TEST_TABLE = "testTombstonesAreCleanedUpCase3";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("case3");
+ Put p = new Put(rowId);
+ p.add(fam, qual, Bytes.toBytes("testValue"));
+ txTable.put(tx1, p);
+ tm.commit(tx1);
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(tx2, d);
+
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertTrue("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertFalse("Delete cell shouldn't be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ assertFalse("Delete shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ }
+
+ /**
+ * Test that when compaction runs, tombstones are cleaned up case4: 1 put (ts < lwm) then tombstone (ts > lwm) not
+ * committed
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreCleanedUpCase4() throws Exception {
+ String TEST_TABLE = "testTombstonesAreCleanedUpCase4";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("case4");
+ Put p = new Put(rowId);
+ p.add(fam, qual, Bytes.toBytes("testValue"));
+ txTable.put(tx1, p);
+ tm.commit(tx1);
+
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(tx2, d);
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertTrue("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Delete cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ assertFalse("Delete shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ }
+
+ /**
+ * Test that when compaction runs, tombstones are cleaned up case5: tombstone (ts < lwm)
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreCleanedUpCase5() throws Exception {
+ String TEST_TABLE = "testTombstonesAreCleanedUpCase5";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ byte[] rowId = Bytes.toBytes("case5");
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(tx1, d);
+ tm.commit(tx1);
+
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertFalse("Delete cell shouldn't be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertFalse("Delete shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ }
+
+ /**
+ * Test that when compaction runs, tombstones are cleaned up case6: tombstone (ts < lwm), then put (ts < lwm)
+ */
+ @Test(timeOut = 60_000)
+ public void testTombstonesAreCleanedUpCase6() throws Exception {
+ String TEST_TABLE = "testTombstonesAreCleanedUpCase6";
+ createTableIfNotExists(TEST_TABLE, Bytes.toBytes(TEST_FAMILY));
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+ byte[] rowId = Bytes.toBytes("case6");
+
+ HBaseTransaction tx1 = (HBaseTransaction) tm.begin();
+ Delete d = new Delete(rowId);
+ d.deleteColumn(fam, qual);
+ txTable.delete(tx1, d);
+ tm.commit(tx1);
+
+ HBaseTransaction tx2 = (HBaseTransaction) tm.begin();
+ Put p = new Put(rowId);
+ p.add(fam, qual, Bytes.toBytes("testValue"));
+ txTable.put(tx2, p);
+ tm.commit(tx2);
+
+ HBaseTransaction lwmTx = (HBaseTransaction) tm.begin();
+ compactWithLWM(lwmTx.getStartTimestamp(), TEST_TABLE);
+
+ TTableCellGetterAdapter getter = new TTableCellGetterAdapter(txTable);
+ assertFalse("Delete cell shouldn't be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertFalse("Delete shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx1.getStartTimestamp(), getter));
+ assertTrue("Put cell should be there",
+ CellUtils.hasCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ assertTrue("Put shadow cell shouldn't be there",
+ CellUtils.hasShadowCell(rowId, fam, qual,
+ tx2.getStartTimestamp(), getter));
+ }
+
+ private void setCompactorLWM(long lwm, String tableName) throws Exception {
+ OmidCompactor omidCompactor = (OmidCompactor) hbaseCluster.getRegions(Bytes.toBytes(tableName)).get(0)
+ .getCoprocessorHost().findCoprocessor(OmidCompactor.class.getName());
+ CommitTable commitTable = injector.getInstance(CommitTable.class);
+ CommitTable.Client commitTableClient = spy(commitTable.getClient());
+ SettableFuture<Long> f = SettableFuture.create();
+ f.set(lwm);
+ doReturn(f).when(commitTableClient).readLowWatermark();
+ omidCompactor.commitTableClientQueue.add(commitTableClient);
+ }
+
+ private void compactEverything(String tableName) throws Exception {
+ compactWithLWM(Long.MAX_VALUE, tableName);
+ }
+
+ private void compactWithLWM(long lwm, String tableName) throws Exception {
+ admin.flush(tableName);
+
+ LOG.info("Regions in table {}: {}", tableName, hbaseCluster.getRegions(Bytes.toBytes(tableName)).size());
+ setCompactorLWM(lwm, tableName);
+ LOG.info("Compacting table {}", tableName);
+ admin.majorCompact(tableName);
+
+ LOG.info("Sleeping for 3 secs");
+ Thread.sleep(3000);
+ LOG.info("Waking up after 3 secs");
+ }
+
+ private long rowCount(String tableName, byte[] family) throws Throwable {
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return aggregationClient.rowCount(TableName.valueOf(tableName), new LongColumnInterpreter(), scan);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompactorScanner.java
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompactorScanner.java b/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompactorScanner.java
new file mode 100644
index 0000000..b16aec3
--- /dev/null
+++ b/hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompactorScanner.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.Client;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.CompactorScanner;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.util.Queue;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+
+public class TestCompactorScanner {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestCompactorScanner.class);
+
+ private static final long TEST_TS = 1L;
+
+ @DataProvider(name = "cell-retain-options")
+ public Object[][] createCellRetainOptions() {
+ return new Object[][]{
+ {1, true}, {2, false},
+ };
+ }
+
+ @Test(dataProvider = "cell-retain-options")
+ public void testShouldRetainNonTransactionallyDeletedCellMethod(int optionIdx, boolean retainOption)
+ throws Exception {
+
+ // Create required mocks
+ @SuppressWarnings("unchecked")
+ ObserverContext<RegionCoprocessorEnvironment> ctx = mock(ObserverContext.class);
+ InternalScanner internalScanner = mock(InternalScanner.class);
+ CommitTable.Client ctClient = mock(CommitTable.Client.class);
+ @SuppressWarnings("unchecked")
+ Queue<Client> queue = mock(Queue.class);
+ RegionCoprocessorEnvironment rce = mock(RegionCoprocessorEnvironment.class);
+ HRegion hRegion = mock(HRegion.class);
+ HRegionInfo hRegionInfo = mock(HRegionInfo.class);
+ SettableFuture<Long> f = SettableFuture.<Long>create();
+
+ // Wire required mock internals
+ f.set(TEST_TS);
+ when(ctClient.readLowWatermark()).thenReturn(f);
+ when(ctx.getEnvironment()).thenReturn(rce);
+ when(rce.getRegion()).thenReturn(hRegion);
+ when(hRegion.getRegionInfo()).thenReturn(hRegionInfo);
+
+ LOG.info("Testing when retain is {}", retainOption);
+ try (CompactorScanner scanner = spy(new CompactorScanner(ctx,
+ internalScanner,
+ ctClient,
+ queue,
+ false,
+ retainOption))) {
+
+ // Different cell types to test
+ KeyValue regularKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.Put);
+ KeyValue deleteKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.Delete);
+ KeyValue deleteColumnKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteColumn);
+ KeyValue deleteFamilyKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteFamily);
+ KeyValue deleteFamilyVersionKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteFamilyVersion);
+
+ assertFalse(scanner.shouldRetainNonTransactionallyDeletedCell(regularKV));
+ assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteKV));
+ assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteColumnKV));
+ assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteFamilyKV));
+ assertEquals(retainOption, scanner.shouldRetainNonTransactionallyDeletedCell(deleteFamilyVersionKV));
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-coprocessor/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/hbase-coprocessor/src/test/resources/log4j.xml b/hbase-coprocessor/src/test/resources/log4j.xml
index 225b90c..c3e535a 100755
--- a/hbase-coprocessor/src/test/resources/log4j.xml
+++ b/hbase-coprocessor/src/test/resources/log4j.xml
@@ -45,11 +45,11 @@
</layout>
</appender>
- <logger name="com.yahoo.omid">
+ <logger name="org.apache.omid">
<level value="TRACE"/>
</logger>
- <logger name="com.yahoo.omid.util">
+ <logger name="org.apache.omid.util">
<level value="INFO"/>
</logger>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/hbase-0/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/pom.xml b/hbase-shims/hbase-0/pom.xml
index 401b2e4..b5c9044 100644
--- a/hbase-shims/hbase-0/pom.xml
+++ b/hbase-shims/hbase-0/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java b/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
deleted file mode 100644
index fb3c690..0000000
--- a/hbase-shims/hbase-0/src/main/java/com/yahoo/omid/HBaseShims.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.Region;
-
-import java.io.IOException;
-
-public class HBaseShims {
-
- static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) {
-
- kv.setMvccVersion(sequenceId);
-
- }
-
- static public Region getRegionCoprocessorRegion(RegionCoprocessorEnvironment env) {
-
- return new Region(env.getRegion());
-
- }
-
- static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException {
-
- for (HRegion r : regionServer.getOnlineRegions(tableName)) {
- r.flushcache();
- }
-
- }
-
- static public void addFamilyToHTableDescriptor(HTableDescriptor tableDesc, HColumnDescriptor columnDesc) {
-
- tableDesc.addFamily(columnDesc);
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/hbase-0/src/main/java/org/apache/omid/HBaseShims.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-0/src/main/java/org/apache/omid/HBaseShims.java b/hbase-shims/hbase-0/src/main/java/org/apache/omid/HBaseShims.java
new file mode 100644
index 0000000..e440ba6
--- /dev/null
+++ b/hbase-shims/hbase-0/src/main/java/org/apache/omid/HBaseShims.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+
+import java.io.IOException;
+
+public class HBaseShims {
+
+ static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) {
+
+ kv.setMvccVersion(sequenceId);
+
+ }
+
+ static public Region getRegionCoprocessorRegion(RegionCoprocessorEnvironment env) {
+
+ return new Region(env.getRegion());
+
+ }
+
+ static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException {
+
+ for (HRegion r : regionServer.getOnlineRegions(tableName)) {
+ r.flushcache();
+ }
+
+ }
+
+ static public void addFamilyToHTableDescriptor(HTableDescriptor tableDesc, HColumnDescriptor columnDesc) {
+
+ tableDesc.addFamily(columnDesc);
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/hbase-1/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/pom.xml b/hbase-shims/hbase-1/pom.xml
index f3de6f4..11dd838 100644
--- a/hbase-shims/hbase-1/pom.xml
+++ b/hbase-shims/hbase-1/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid-shims-aggregator</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java b/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
deleted file mode 100644
index 6e54cee..0000000
--- a/hbase-shims/hbase-1/src/main/java/com/yahoo/omid/HBaseShims.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid;
-
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
-import org.apache.hadoop.hbase.regionserver.Region;
-
-import java.io.IOException;
-
-public class HBaseShims {
-
- static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) {
-
- kv.setSequenceId(sequenceId);
-
- }
-
- static public Region getRegionCoprocessorRegion(RegionCoprocessorEnvironment env) {
-
- return env.getRegion();
-
- }
-
- static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException {
-
- for (Region r : regionServer.getOnlineRegions(tableName)) {
- r.flush(true);
- }
-
- }
-
- static public void addFamilyToHTableDescriptor(HTableDescriptor tableDesc, HColumnDescriptor columnDesc) {
-
- tableDesc.addFamily(columnDesc);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/hbase-1/src/main/java/org/apache/omid/HBaseShims.java
----------------------------------------------------------------------
diff --git a/hbase-shims/hbase-1/src/main/java/org/apache/omid/HBaseShims.java b/hbase-shims/hbase-1/src/main/java/org/apache/omid/HBaseShims.java
new file mode 100644
index 0000000..052329a
--- /dev/null
+++ b/hbase-shims/hbase-1/src/main/java/org/apache/omid/HBaseShims.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.Region;
+
+import java.io.IOException;
+
+public class HBaseShims {
+
+ static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) {
+
+ kv.setSequenceId(sequenceId);
+
+ }
+
+ static public Region getRegionCoprocessorRegion(RegionCoprocessorEnvironment env) {
+
+ return env.getRegion();
+
+ }
+
+ static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException {
+
+ for (Region r : regionServer.getOnlineRegions(tableName)) {
+ r.flush(true);
+ }
+
+ }
+
+ static public void addFamilyToHTableDescriptor(HTableDescriptor tableDesc, HColumnDescriptor columnDesc) {
+
+ tableDesc.addFamily(columnDesc);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-shims/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-shims/pom.xml b/hbase-shims/pom.xml
index d04bab3..32576f2 100644
--- a/hbase-shims/pom.xml
+++ b/hbase-shims/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-tools/pom.xml b/hbase-tools/pom.xml
index 3010241..a43fadc 100644
--- a/hbase-tools/pom.xml
+++ b/hbase-tools/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -17,13 +17,13 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>timestamp-storage</artifactId>
<version>${project.version}</version>
</dependency>
@@ -86,7 +86,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -97,7 +97,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
[41/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java b/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
deleted file mode 100644
index 13efb50..0000000
--- a/examples/src/main/java/com/yahoo/omid/examples/ConfigurationExample.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.examples;
-
-import com.yahoo.omid.transaction.HBaseOmidClientConfiguration;
-import com.yahoo.omid.transaction.HBaseTransactionManager;
-import com.yahoo.omid.transaction.RollbackException;
-import com.yahoo.omid.transaction.TTable;
-import com.yahoo.omid.transaction.Transaction;
-import com.yahoo.omid.transaction.TransactionException;
-import com.yahoo.omid.transaction.TransactionManager;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-import static com.yahoo.omid.tso.client.OmidClientConfiguration.ConnType.DIRECT;
-
-/**
- * ****************************************************************************************************************
- *
- * This example code demonstrates different ways to configure the Omid client settings for HBase
- *
- * ****************************************************************************************************************
- *
- * Please @see{BasicExample} first on how to use with all default settings
- *
- */
-public class ConfigurationExample {
- private static final Logger LOG = LoggerFactory.getLogger(ConfigurationExample.class);
-
- public static void main(String[] args) throws Exception {
-
- LOG.info("Parsing command line arguments");
- String userTableName = "MY_TX_TABLE";
- if (args != null && args.length > 0 && StringUtils.isNotEmpty(args[0])) {
- userTableName = args[0];
- }
- byte[] family = Bytes.toBytes("MY_CF");
- if (args != null && args.length > 1 && StringUtils.isNotEmpty(args[1])) {
- family = Bytes.toBytes(args[1]);
- }
- LOG.info("Table '{}', column family '{}'", userTableName, Bytes.toString(family));
-
- ConfigurationExample example = new ConfigurationExample();
-
- // -----------------------------------------------------------------------------------------------------------
- // Omid client settings configuration through the 'hbase-omid-client-config.yml' configuration file
- // -----------------------------------------------------------------------------------------------------------
- // The HBaseOmidClientConfiguration loads defaults from 'default-hbase-omid-client-config.yml'
- // and then also applies settings from 'hbase-omid-client-config.yml' if it's available in the classpath.
- // In the code snippet below, the user settings are loaded from the 'hbase-omid-client-config.yml' file in
- // the /conf directory that is included in the example classpath (See run.sh.) You can modify the Omid client
- // settings there or you can place your own 'hbase-omid-client-config.yml' file with all your custom settings
- // in the application classpath.
-
- example.doWork(userTableName, family, new HBaseOmidClientConfiguration());
-
- // -----------------------------------------------------------------------------------------------------------
- // Omid client settings configuration from application code
- // -----------------------------------------------------------------------------------------------------------
- // You can also configure Omid programmatically from your code. This is useful for example in unit tests.
- // The HBaseOmidClientConfiguration still loads defaults from 'default-hbase-omid-client-config.yml' first,
- // and then applies settings from 'hbase-omid-client-config.yml' if it's available and then use explicit
- // settings in the code. An example of an explicit Omid client configuration in code is shown below.
-
- HBaseOmidClientConfiguration omidClientConfiguration = new HBaseOmidClientConfiguration();
- omidClientConfiguration.setConnectionType(DIRECT);
- omidClientConfiguration.setConnectionString("localhost:54758");
- omidClientConfiguration.setRetryDelayInMs(3000);
-
- example.doWork(userTableName, family, omidClientConfiguration);
- }
-
- private void doWork(String userTableName, byte[] family, HBaseOmidClientConfiguration configuration)
- throws IOException, TransactionException, RollbackException, InterruptedException {
-
- byte[] exampleRow1 = Bytes.toBytes("EXAMPLE_ROW1");
- byte[] exampleRow2 = Bytes.toBytes("EXAMPLE_ROW2");
- byte[] qualifier = Bytes.toBytes("MY_Q");
- byte[] dataValue1 = Bytes.toBytes("val1");
- byte[] dataValue2 = Bytes.toBytes("val2");
-
- LOG.info("Creating access to Omid Transaction Manager & Transactional Table '{}'", userTableName);
- try (TransactionManager tm = HBaseTransactionManager.newInstance(configuration);
- TTable txTable = new TTable(userTableName))
- {
- for (int i = 0; i < 100; i++) {
- Transaction tx = tm.begin();
- LOG.info("Transaction #{} {} STARTED", i, tx);
-
- Put row1 = new Put(exampleRow1);
- row1.add(family, qualifier, dataValue1);
- txTable.put(tx, row1);
- LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
- tx, userTableName, Bytes.toString(exampleRow1), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue1));
-
- Put row2 = new Put(exampleRow2);
- row2.add(family, qualifier, dataValue2);
- txTable.put(tx, row2);
- LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
- tx, userTableName, Bytes.toString(exampleRow2), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue2));
-
- tm.commit(tx);
- LOG.info("Transaction #{} {} COMMITTED", i, tx);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java b/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
deleted file mode 100644
index 81ed552..0000000
--- a/examples/src/main/java/com/yahoo/omid/examples/SnapshotIsolationExample.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.examples;
-
-import com.yahoo.omid.transaction.HBaseTransactionManager;
-import com.yahoo.omid.transaction.RollbackException;
-import com.yahoo.omid.transaction.TTable;
-import com.yahoo.omid.transaction.Transaction;
-import com.yahoo.omid.transaction.TransactionManager;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-
-/**
- * ****************************************************************************************************************
- *
- * Example code which demonstrates the preservation of Snapshot Isolation when writing shared data concurrently
- *
- * ****************************************************************************************************************
- *
- * Please @see{BasicExample} first
- *
- * In the code below, two concurrent transactions (Tx1 & Tx2), try to update the same column in HBase. This will result
- * in the rollback of Tx2 -the last one trying to commit- due to conflicts in the writeset with the previously
- * committed transaction Tx1. Also shows how Tx2 reads the right values from its own snapshot in HBase data.
- *
- * After building the package with 'mvn clean package' find the resulting examples-<version>-bin.tar.gz file in the
- * 'examples/target' folder. Copy it to the target host and expand with 'tar -zxvf examples-<version>-bin.tar.gz'.
- *
- * Make sure that 'hbase-site.xml' and 'core-site.xml' are either in classpath (see run.sh) or explicitly referenced via
- * command line arguments. If a secure HBase deployment is needed, use also command line arguments to specify the
- * principal (user) and keytab file.
- *
- * The example requires a user table to perform transactional read/write operations. A table is already specified in
- * the default configuration, and can be created with the following command using the 'hbase shell':
- *
- * <pre>
- * create 'MY_TX_TABLE', {NAME => 'MY_CF', VERSIONS => '2147483647', TTL => '2147483647'}
- * </pre>
- *
- * Make sure that the principal/user has RW permissions for the given table using also the 'hbase shell':
- * <pre>
- * grant '<principal/user>', 'RW', 'MY_TX_TABLE'
- * </pre>
- *
- * Alternatively, a table with a column family already created can be used by specifying the table name and column
- * family identifiers using the command line arguments (see details also in 'run.sh') If a table namespace is required,
- * specify it like this: 'namespace:table_name'
- *
- * Finally, run the example using the 'run.sh' script without arguments or specifying the necessary configuration
- * parameters if required.
- */
-public class SnapshotIsolationExample {
-
- private static final Logger LOG = LoggerFactory.getLogger(SnapshotIsolationExample.class);
-
- public static void main(String[] args) throws Exception {
-
- LOG.info("Parsing the command line arguments");
- String userTableName = "MY_TX_TABLE";
- if (args != null && args.length > 0 && StringUtils.isNotEmpty(args[0])) {
- userTableName = args[0];
- }
- byte[] family = Bytes.toBytes("MY_CF");
- if (args != null && args.length > 1 && StringUtils.isNotEmpty(args[1])) {
- family = Bytes.toBytes(args[1]);
- }
- LOG.info("Table '{}', column family '{}'", userTableName, Bytes.toString(family));
-
- byte[] exampleRow = Bytes.toBytes("EXAMPLE_ROW");
- byte[] qualifier = Bytes.toBytes("MY_Q");
- byte[] initialData = Bytes.toBytes("initialVal");
- byte[] dataValue1 = Bytes.toBytes("val1");
- byte[] dataValue2 = Bytes.toBytes("val2");
-
- LOG.info("--------------------------------------------------------------------------------------------------");
- LOG.info("NOTE: All Transactions in the Example access column {}:{}/{}/{} [TABLE:ROW/CF/Q]",
- userTableName, Bytes.toString(exampleRow), Bytes.toString(family), Bytes.toString(qualifier));
- LOG.info("--------------------------------------------------------------------------------------------------");
-
- LOG.info("Creating access to Omid Transaction Manager & Transactional Table '{}'", userTableName);
- try (TransactionManager tm = HBaseTransactionManager.newInstance();
- TTable txTable = new TTable(userTableName))
- {
-
- // A transaction Tx0 sets an initial value to a particular column in an specific row
- Transaction tx0 = tm.begin();
- Put initialPut = new Put(exampleRow);
- initialPut.add(family, qualifier, initialData);
- txTable.put(tx0, initialPut);
- tm.commit(tx0);
- LOG.info("Initial Transaction {} COMMITTED. Base value written in {}:{}/{}/{} = {}",
- tx0, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(initialData));
-
- // Transaction Tx1 starts, creates its own snapshot of the current data in HBase and writes new data
- Transaction tx1 = tm.begin();
- LOG.info("Transaction {} STARTED", tx1);
- Put tx1Put = new Put(exampleRow);
- tx1Put.add(family, qualifier, dataValue1);
- txTable.put(tx1, tx1Put);
- LOG.info("Transaction {} updates base value in {}:{}/{}/{} = {} in its own Snapshot",
- tx1, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue1));
-
- // A concurrent transaction Tx2 starts, creates its own snapshot and reads the column value
- Transaction tx2 = tm.begin();
- LOG.info("Concurrent Transaction {} STARTED", tx2);
- Get tx2Get = new Get(exampleRow);
- tx2Get.addColumn(family, qualifier);
- // As Tx1 is not yet committed, it should read the value set by Tx0 not the value written by Tx1
- Result tx2GetResult = txTable.get(tx2, tx2Get);
- assert Arrays.equals(tx2GetResult.value(), initialData);
- LOG.info("Concurrent Transaction {} should read base value in {}:{}/{}/{} from its Snapshot | Value read = {}",
- tx2, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(tx2GetResult.value()));
-
- // Transaction Tx1 tries to commit and as there're no conflicting changes, persists the new value in HBase
- tm.commit(tx1);
- LOG.info("Transaction {} COMMITTED. New column value {}:{}/{}/{} = {}",
- tx1, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue1));
-
- // Tx2 reading again after Tx1 commit must read data from its snapshot...
- tx2Get = new Get(exampleRow);
- tx2Get.addColumn(family, qualifier);
- tx2GetResult = txTable.get(tx2, tx2Get);
- // ...so it must read the initial value written by Tx0
- LOG.info("Concurrent Transaction {} should read again base value in {}:{}/{}/{} from its Snapshot | Value read = {}",
- tx2, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(tx2GetResult.value()));
-
- // Tx2 tries to write the column written by the committed concurrent transaction Tx1...
- Put tx2Put = new Put(exampleRow);
- tx2Put.add(family, qualifier, dataValue2);
- txTable.put(tx2, tx2Put);
- LOG.info("Concurrent Transaction {} updates {}:{}/{}/{} = {} in its own Snapshot (Will conflict with {} at commit time)",
- tx2, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
- Bytes.toString(qualifier), Bytes.toString(dataValue1), tx1);
-
- // ... and when committing, Tx2 has to abort due to concurrent conflicts with committed transaction Tx1
- try {
- LOG.info("Concurrent Transaction {} TRYING TO COMMIT", tx2);
- tm.commit(tx2);
- } catch (RollbackException e) {
- LOG.error("Concurrent Transaction {} ROLLED-BACK!!! : {}", tx2, e.getMessage());
- }
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/java/org/apache/omid/examples/BasicExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/omid/examples/BasicExample.java b/examples/src/main/java/org/apache/omid/examples/BasicExample.java
new file mode 100644
index 0000000..2d153e8
--- /dev/null
+++ b/examples/src/main/java/org/apache/omid/examples/BasicExample.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.examples;
+
+import org.apache.omid.transaction.HBaseTransactionManager;
+import org.apache.omid.transaction.TTable;
+import org.apache.omid.transaction.Transaction;
+import org.apache.omid.transaction.TransactionManager;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * ****************************************************************************************************
+ *
+ * Example code which demonstrates an atomic write into two different rows in HBase
+ *
+ * ****************************************************************************************************
+ *
+ * After building the package with 'mvn clean package' find the resulting examples-<version>-bin.tar.gz file in the
+ * 'examples/target' folder. Copy it to the target host and expand with 'tar -zxvf examples-<version>-bin.tar.gz'.
+ *
+ * Make sure that 'hbase-site.xml' and 'core-site.xml' are either in classpath (see run.sh) or explicitly referenced in
+ * configuration file. If a secure HBase deployment is needed, make sure to specify the principal (user) and keytab file.
+ *
+ * The example requires a user table to perform transactional read/write operations. A table is already specified in
+ * the default configuration, and can be created with the following command using the 'hbase shell':
+ *
+ * <pre>
+ * create 'MY_TX_TABLE', {NAME => 'MY_CF', VERSIONS => '2147483647', TTL => '2147483647'}
+ * </pre>
+ *
+ * Make sure that the principal/user has RW permissions for the given table using also the 'hbase shell':
+ * <pre>
+ * grant '<principal/user>', 'RW', 'MY_TX_TABLE'
+ * </pre>
+ *
+ * Alternatively, a table with a column family already created can be used by specifying the table name and column
+ * family identifiers using the command line arguments (see details also in 'run.sh') If a table namespace is required,
+ * specify it like this: 'namespace:table_name'
+ *
+ * Finally, run the example using the 'run.sh' script without arguments or specifying the necessary configuration
+ * parameters.
+ */
+public class BasicExample {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BasicExample.class);
+
+ public static void main(String[] args) throws Exception {
+
+ LOG.info("Parsing command line arguments");
+ String userTableName = "MY_TX_TABLE";
+ if (args != null && args.length > 0 && StringUtils.isNotEmpty(args[0])) {
+ userTableName = args[0];
+ }
+ byte[] family = Bytes.toBytes("MY_CF");
+ if (args != null && args.length > 1 && StringUtils.isNotEmpty(args[1])) {
+ family = Bytes.toBytes(args[1]);
+ }
+ LOG.info("Table '{}', column family '{}'", userTableName, Bytes.toString(family));
+
+ byte[] exampleRow1 = Bytes.toBytes("EXAMPLE_ROW1");
+ byte[] exampleRow2 = Bytes.toBytes("EXAMPLE_ROW2");
+ byte[] qualifier = Bytes.toBytes("MY_Q");
+ byte[] dataValue1 = Bytes.toBytes("val1");
+ byte[] dataValue2 = Bytes.toBytes("val2");
+
+ LOG.info("Creating access to Omid Transaction Manager & Transactional Table '{}'", userTableName);
+ try (TransactionManager tm = HBaseTransactionManager.newInstance();
+ TTable txTable = new TTable(userTableName))
+ {
+ Transaction tx = tm.begin();
+ LOG.info("Transaction {} STARTED", tx);
+
+ Put row1 = new Put(exampleRow1);
+ row1.add(family, qualifier, dataValue1);
+ txTable.put(tx, row1);
+ LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
+ tx, userTableName, Bytes.toString(exampleRow1), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue1));
+
+ Put row2 = new Put(exampleRow2);
+ row2.add(family, qualifier, dataValue2);
+ txTable.put(tx, row2);
+ LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
+ tx, userTableName, Bytes.toString(exampleRow2), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue2));
+
+ tm.commit(tx);
+ LOG.info("Transaction {} COMMITTED", tx);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/java/org/apache/omid/examples/ConfigurationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/omid/examples/ConfigurationExample.java b/examples/src/main/java/org/apache/omid/examples/ConfigurationExample.java
new file mode 100644
index 0000000..b15ac8a
--- /dev/null
+++ b/examples/src/main/java/org/apache/omid/examples/ConfigurationExample.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.examples;
+
+import org.apache.omid.transaction.HBaseOmidClientConfiguration;
+import org.apache.omid.transaction.HBaseTransactionManager;
+import org.apache.omid.transaction.RollbackException;
+import org.apache.omid.transaction.TTable;
+import org.apache.omid.transaction.Transaction;
+import org.apache.omid.transaction.TransactionException;
+import org.apache.omid.transaction.TransactionManager;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.omid.tso.client.OmidClientConfiguration.ConnType.DIRECT;
+
+/**
+ * ****************************************************************************************************************
+ *
+ * This example code demonstrates different ways to configure the Omid client settings for HBase
+ *
+ * ****************************************************************************************************************
+ *
+ * Please @see{BasicExample} first on how to use with all default settings
+ *
+ */
+public class ConfigurationExample {
+ private static final Logger LOG = LoggerFactory.getLogger(ConfigurationExample.class);
+
+ public static void main(String[] args) throws Exception {
+
+ LOG.info("Parsing command line arguments");
+ String userTableName = "MY_TX_TABLE";
+ if (args != null && args.length > 0 && StringUtils.isNotEmpty(args[0])) {
+ userTableName = args[0];
+ }
+ byte[] family = Bytes.toBytes("MY_CF");
+ if (args != null && args.length > 1 && StringUtils.isNotEmpty(args[1])) {
+ family = Bytes.toBytes(args[1]);
+ }
+ LOG.info("Table '{}', column family '{}'", userTableName, Bytes.toString(family));
+
+ ConfigurationExample example = new ConfigurationExample();
+
+ // -----------------------------------------------------------------------------------------------------------
+ // Omid client settings configuration through the 'hbase-omid-client-config.yml' configuration file
+ // -----------------------------------------------------------------------------------------------------------
+ // The HBaseOmidClientConfiguration loads defaults from 'default-hbase-omid-client-config.yml'
+ // and then also applies settings from 'hbase-omid-client-config.yml' if it's available in the classpath.
+ // In the code snippet below, the user settings are loaded from the 'hbase-omid-client-config.yml' file in
+ // the /conf directory that is included in the example classpath (See run.sh.) You can modify the Omid client
+ // settings there or you can place your own 'hbase-omid-client-config.yml' file with all your custom settings
+ // in the application classpath.
+
+ example.doWork(userTableName, family, new HBaseOmidClientConfiguration());
+
+ // -----------------------------------------------------------------------------------------------------------
+ // Omid client settings configuration from application code
+ // -----------------------------------------------------------------------------------------------------------
+ // You can also configure Omid programmatically from your code. This is useful for example in unit tests.
+ // The HBaseOmidClientConfiguration still loads defaults from 'default-hbase-omid-client-config.yml' first,
+ // and then applies settings from 'hbase-omid-client-config.yml' if it's available and then use explicit
+ // settings in the code. An example of an explicit Omid client configuration in code is shown below.
+
+ HBaseOmidClientConfiguration omidClientConfiguration = new HBaseOmidClientConfiguration();
+ omidClientConfiguration.setConnectionType(DIRECT);
+ omidClientConfiguration.setConnectionString("localhost:54758");
+ omidClientConfiguration.setRetryDelayInMs(3000);
+
+ example.doWork(userTableName, family, omidClientConfiguration);
+ }
+
+ private void doWork(String userTableName, byte[] family, HBaseOmidClientConfiguration configuration)
+ throws IOException, TransactionException, RollbackException, InterruptedException {
+
+ byte[] exampleRow1 = Bytes.toBytes("EXAMPLE_ROW1");
+ byte[] exampleRow2 = Bytes.toBytes("EXAMPLE_ROW2");
+ byte[] qualifier = Bytes.toBytes("MY_Q");
+ byte[] dataValue1 = Bytes.toBytes("val1");
+ byte[] dataValue2 = Bytes.toBytes("val2");
+
+ LOG.info("Creating access to Omid Transaction Manager & Transactional Table '{}'", userTableName);
+ try (TransactionManager tm = HBaseTransactionManager.newInstance(configuration);
+ TTable txTable = new TTable(userTableName))
+ {
+ for (int i = 0; i < 100; i++) {
+ Transaction tx = tm.begin();
+ LOG.info("Transaction #{} {} STARTED", i, tx);
+
+ Put row1 = new Put(exampleRow1);
+ row1.add(family, qualifier, dataValue1);
+ txTable.put(tx, row1);
+ LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
+ tx, userTableName, Bytes.toString(exampleRow1), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue1));
+
+ Put row2 = new Put(exampleRow2);
+ row2.add(family, qualifier, dataValue2);
+ txTable.put(tx, row2);
+ LOG.info("Transaction {} trying to write a new value in [TABLE:ROW/CF/Q] => {}:{}/{}/{} = {} ",
+ tx, userTableName, Bytes.toString(exampleRow2), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue2));
+
+ tm.commit(tx);
+ LOG.info("Transaction #{} {} COMMITTED", i, tx);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/java/org/apache/omid/examples/SnapshotIsolationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/omid/examples/SnapshotIsolationExample.java b/examples/src/main/java/org/apache/omid/examples/SnapshotIsolationExample.java
new file mode 100644
index 0000000..a98afea
--- /dev/null
+++ b/examples/src/main/java/org/apache/omid/examples/SnapshotIsolationExample.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.examples;
+
+import org.apache.omid.transaction.HBaseTransactionManager;
+import org.apache.omid.transaction.RollbackException;
+import org.apache.omid.transaction.TTable;
+import org.apache.omid.transaction.Transaction;
+import org.apache.omid.transaction.TransactionManager;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+/**
+ * ****************************************************************************************************************
+ *
+ * Example code which demonstrates the preservation of Snapshot Isolation when writing shared data concurrently
+ *
+ * ****************************************************************************************************************
+ *
+ * Please @see{BasicExample} first
+ *
+ * In the code below, two concurrent transactions (Tx1 & Tx2), try to update the same column in HBase. This will result
+ * in the rollback of Tx2 -the last one trying to commit- due to conflicts in the writeset with the previously
+ * committed transaction Tx1. Also shows how Tx2 reads the right values from its own snapshot in HBase data.
+ *
+ * After building the package with 'mvn clean package' find the resulting examples-<version>-bin.tar.gz file in the
+ * 'examples/target' folder. Copy it to the target host and expand with 'tar -zxvf examples-<version>-bin.tar.gz'.
+ *
+ * Make sure that 'hbase-site.xml' and 'core-site.xml' are either in classpath (see run.sh) or explicitly referenced via
+ * command line arguments. If a secure HBase deployment is needed, use also command line arguments to specify the
+ * principal (user) and keytab file.
+ *
+ * The example requires a user table to perform transactional read/write operations. A table is already specified in
+ * the default configuration, and can be created with the following command using the 'hbase shell':
+ *
+ * <pre>
+ * create 'MY_TX_TABLE', {NAME => 'MY_CF', VERSIONS => '2147483647', TTL => '2147483647'}
+ * </pre>
+ *
+ * Make sure that the principal/user has RW permissions for the given table using also the 'hbase shell':
+ * <pre>
+ * grant '<principal/user>', 'RW', 'MY_TX_TABLE'
+ * </pre>
+ *
+ * Alternatively, a table with a column family already created can be used by specifying the table name and column
+ * family identifiers using the command line arguments (see details also in 'run.sh') If a table namespace is required,
+ * specify it like this: 'namespace:table_name'
+ *
+ * Finally, run the example using the 'run.sh' script without arguments or specifying the necessary configuration
+ * parameters if required.
+ */
+public class SnapshotIsolationExample {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SnapshotIsolationExample.class);
+
+ public static void main(String[] args) throws Exception {
+
+ LOG.info("Parsing the command line arguments");
+ String userTableName = "MY_TX_TABLE";
+ if (args != null && args.length > 0 && StringUtils.isNotEmpty(args[0])) {
+ userTableName = args[0];
+ }
+ byte[] family = Bytes.toBytes("MY_CF");
+ if (args != null && args.length > 1 && StringUtils.isNotEmpty(args[1])) {
+ family = Bytes.toBytes(args[1]);
+ }
+ LOG.info("Table '{}', column family '{}'", userTableName, Bytes.toString(family));
+
+ byte[] exampleRow = Bytes.toBytes("EXAMPLE_ROW");
+ byte[] qualifier = Bytes.toBytes("MY_Q");
+ byte[] initialData = Bytes.toBytes("initialVal");
+ byte[] dataValue1 = Bytes.toBytes("val1");
+ byte[] dataValue2 = Bytes.toBytes("val2");
+
+ LOG.info("--------------------------------------------------------------------------------------------------");
+ LOG.info("NOTE: All Transactions in the Example access column {}:{}/{}/{} [TABLE:ROW/CF/Q]",
+ userTableName, Bytes.toString(exampleRow), Bytes.toString(family), Bytes.toString(qualifier));
+ LOG.info("--------------------------------------------------------------------------------------------------");
+
+ LOG.info("Creating access to Omid Transaction Manager & Transactional Table '{}'", userTableName);
+ try (TransactionManager tm = HBaseTransactionManager.newInstance();
+ TTable txTable = new TTable(userTableName))
+ {
+
+ // A transaction Tx0 sets an initial value to a particular column in an specific row
+ Transaction tx0 = tm.begin();
+ Put initialPut = new Put(exampleRow);
+ initialPut.add(family, qualifier, initialData);
+ txTable.put(tx0, initialPut);
+ tm.commit(tx0);
+ LOG.info("Initial Transaction {} COMMITTED. Base value written in {}:{}/{}/{} = {}",
+ tx0, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(initialData));
+
+ // Transaction Tx1 starts, creates its own snapshot of the current data in HBase and writes new data
+ Transaction tx1 = tm.begin();
+ LOG.info("Transaction {} STARTED", tx1);
+ Put tx1Put = new Put(exampleRow);
+ tx1Put.add(family, qualifier, dataValue1);
+ txTable.put(tx1, tx1Put);
+ LOG.info("Transaction {} updates base value in {}:{}/{}/{} = {} in its own Snapshot",
+ tx1, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue1));
+
+ // A concurrent transaction Tx2 starts, creates its own snapshot and reads the column value
+ Transaction tx2 = tm.begin();
+ LOG.info("Concurrent Transaction {} STARTED", tx2);
+ Get tx2Get = new Get(exampleRow);
+ tx2Get.addColumn(family, qualifier);
+ // As Tx1 is not yet committed, it should read the value set by Tx0 not the value written by Tx1
+ Result tx2GetResult = txTable.get(tx2, tx2Get);
+ assert Arrays.equals(tx2GetResult.value(), initialData);
+ LOG.info("Concurrent Transaction {} should read base value in {}:{}/{}/{} from its Snapshot | Value read = {}",
+ tx2, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(tx2GetResult.value()));
+
+ // Transaction Tx1 tries to commit and as there're no conflicting changes, persists the new value in HBase
+ tm.commit(tx1);
+ LOG.info("Transaction {} COMMITTED. New column value {}:{}/{}/{} = {}",
+ tx1, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue1));
+
+ // Tx2 reading again after Tx1 commit must read data from its snapshot...
+ tx2Get = new Get(exampleRow);
+ tx2Get.addColumn(family, qualifier);
+ tx2GetResult = txTable.get(tx2, tx2Get);
+ // ...so it must read the initial value written by Tx0
+ LOG.info("Concurrent Transaction {} should read again base value in {}:{}/{}/{} from its Snapshot | Value read = {}",
+ tx2, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(tx2GetResult.value()));
+
+ // Tx2 tries to write the column written by the committed concurrent transaction Tx1...
+ Put tx2Put = new Put(exampleRow);
+ tx2Put.add(family, qualifier, dataValue2);
+ txTable.put(tx2, tx2Put);
+ LOG.info("Concurrent Transaction {} updates {}:{}/{}/{} = {} in its own Snapshot (Will conflict with {} at commit time)",
+ tx2, userTableName, Bytes.toString(exampleRow), Bytes.toString(family),
+ Bytes.toString(qualifier), Bytes.toString(dataValue1), tx1);
+
+ // ... and when committing, Tx2 has to abort due to concurrent conflicts with committed transaction Tx1
+ try {
+ LOG.info("Concurrent Transaction {} TRYING TO COMMIT", tx2);
+ tm.commit(tx2);
+ } catch (RollbackException e) {
+ LOG.error("Concurrent Transaction {} ROLLED-BACK!!! : {}", tx2, e.getMessage());
+ }
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/resources/hbase-omid-client-config.yml
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/hbase-omid-client-config.yml b/examples/src/main/resources/hbase-omid-client-config.yml
index 3314d5d..cb1c08b 100644
--- a/examples/src/main/resources/hbase-omid-client-config.yml
+++ b/examples/src/main/resources/hbase-omid-client-config.yml
@@ -3,15 +3,15 @@
# principal: omid_hbase_client
# keytab: /path/to/hbase/client/keytab
-omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration
+omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration
connectionString: "localhost:54758"
-metrics: !!com.yahoo.omid.metrics.CodahaleMetricsProvider [
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig {
+metrics: !!org.apache.omid.metrics.CodahaleMetricsProvider [
+ !!org.apache.omid.metrics.CodahaleMetricsConfig {
outputFreqInSecs: 1,
reporters: !!set {
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
- !!com.yahoo.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CSV,
+ !!org.apache.omid.metrics.CodahaleMetricsConfig$Reporter CONSOLE
},
csvDir: "csvMetrics",
prefix: "somePrefix",
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/examples/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/log4j.properties b/examples/src/main/resources/log4j.properties
index f65f0d9..ebb2e68 100644
--- a/examples/src/main/resources/log4j.properties
+++ b/examples/src/main/resources/log4j.properties
@@ -1,6 +1,5 @@
log4j.rootLogger=INFO,console
-
-log4j.logger.com.yahoo.omid.examples=INFO
+log4j.logger.org.apache.omid.examples=INFO
# Logging Threshold
log4j.threshold=ALL
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-client/pom.xml b/hbase-client/pom.xml
index cab62e8..1b2e25c 100644
--- a/hbase-client/pom.xml
+++ b/hbase-client/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
@@ -18,37 +18,37 @@
<!-- Dependencies on Omid modules -->
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>transaction-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-commit-table</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>tso-server</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>tso-server</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>commit-table</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<classifier>tests</classifier>
</dependency>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase-tools</artifactId>
<version>${project.version}</version>
<scope>test</scope>
@@ -159,7 +159,7 @@
</activation>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase0-shims</artifactId>
<version>${project.version}</version>
</dependency>
@@ -170,7 +170,7 @@
<id>hbase-1</id>
<dependencies>
<dependency>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>hbase1-shims</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
deleted file mode 100644
index 2e6a94f..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/ColumnWrapper.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import java.util.Arrays;
-
-public class ColumnWrapper {
- private byte[] family;
- private byte[] qualifier;
-
- public ColumnWrapper(byte[] family, byte[] qualifier) {
- this.family = family;
- this.qualifier = qualifier;
- }
-
- public byte[] getFamily() {
- return family;
- }
-
- public byte[] getQualifier() {
- return qualifier;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + Arrays.hashCode(family);
- result = prime * result + Arrays.hashCode(qualifier);
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- ColumnWrapper other = (ColumnWrapper) obj;
- if (!Arrays.equals(family, other.family))
- return false;
- if (!Arrays.equals(qualifier, other.qualifier))
- return false;
- return true;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
deleted file mode 100644
index 8055a03..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseAsyncPostCommitter.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.yahoo.omid.tso.client.CellId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.Callable;
-
-public class HBaseAsyncPostCommitter implements PostCommitActions {
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseAsyncPostCommitter.class);
-
- private PostCommitActions syncPostCommitter;
-
- private ListeningExecutorService postCommitExecutor;
-
- public HBaseAsyncPostCommitter(PostCommitActions postCommitter, ListeningExecutorService postCommitExecutor) {
- this.syncPostCommitter = postCommitter;
- this.postCommitExecutor = postCommitExecutor;
- }
-
- @Override
- public ListenableFuture<Void> updateShadowCells(final AbstractTransaction<? extends CellId> transaction) {
-
- return postCommitExecutor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- syncPostCommitter.updateShadowCells(transaction);
- return null;
- }
-
- });
-
- }
-
- @Override
- public ListenableFuture<Void> removeCommitTableEntry(final AbstractTransaction<? extends CellId> transaction) {
-
- return postCommitExecutor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- syncPostCommitter.removeCommitTableEntry(transaction);
- return null;
- }
- });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
deleted file mode 100644
index ca9d675..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseCellId.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.hash.Hashing;
-import com.yahoo.omid.tso.client.CellId;
-import org.apache.hadoop.hbase.client.HTableInterface;
-
-import static com.google.common.base.Charsets.UTF_8;
-
-public class HBaseCellId implements CellId {
-
- private final HTableInterface table;
- private final byte[] row;
- private final byte[] family;
- private final byte[] qualifier;
- private long timestamp;
-
- public HBaseCellId(HTableInterface table, byte[] row, byte[] family, byte[] qualifier, long timestamp) {
- this.timestamp = timestamp;
- this.table = table;
- this.row = row;
- this.family = family;
- this.qualifier = qualifier;
- }
-
- public HTableInterface getTable() {
- return table;
- }
-
- public byte[] getRow() {
- return row;
- }
-
- public byte[] getFamily() {
- return family;
- }
-
- public byte[] getQualifier() {
- return qualifier;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public String toString() {
- return new String(table.getTableName(), UTF_8)
- + ":" + new String(row, UTF_8)
- + ":" + new String(family, UTF_8)
- + ":" + new String(qualifier, UTF_8)
- + ":" + timestamp;
- }
-
- @Override
- public long getCellId() {
- return Hashing.murmur3_128().newHasher()
- .putBytes(table.getTableName())
- .putBytes(row)
- .putBytes(family)
- .putBytes(qualifier)
- .hash().asLong();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
deleted file mode 100644
index 19c90cb..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseOmidClientConfiguration.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
-import com.yahoo.omid.YAMLUtils;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.tools.hbase.SecureHBaseConfig;
-import com.yahoo.omid.tso.client.OmidClientConfiguration.PostCommitMode;
-import com.yahoo.omid.tso.client.OmidClientConfiguration;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-
-/**
- * Configuration for HBase's Omid client side
- */
-public class HBaseOmidClientConfiguration extends SecureHBaseConfig {
-
- private static final String DEFAULT_CONFIG_FILE_NAME = "default-hbase-omid-client-config.yml";
- private static final String CONFIG_FILE_NAME = "hbase-omid-client-config.yml";
- private Configuration hbaseConfiguration = HBaseConfiguration.create();
- private String commitTableName;
- @Inject
- private OmidClientConfiguration omidClientConfiguration;
- private MetricsRegistry metrics;
-
- // ----------------------------------------------------------------------------------------------------------------
- // Instantiation
- // ----------------------------------------------------------------------------------------------------------------
- public HBaseOmidClientConfiguration() {
- this(CONFIG_FILE_NAME);
- }
-
- @VisibleForTesting
- HBaseOmidClientConfiguration(String configFileName) {
- new YAMLUtils().loadSettings(configFileName, DEFAULT_CONFIG_FILE_NAME, this);
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Getters and setters for config params
- // ----------------------------------------------------------------------------------------------------------------
-
- public Configuration getHBaseConfiguration() {
- return hbaseConfiguration;
- }
-
- public void setHBaseConfiguration(Configuration hbaseConfiguration) {
- this.hbaseConfiguration = hbaseConfiguration;
- }
-
- public PostCommitMode getPostCommitMode() {
- return omidClientConfiguration.getPostCommitMode();
- }
-
- public void setPostCommitMode(PostCommitMode postCommitMode) {
- omidClientConfiguration.setPostCommitMode(postCommitMode);
- }
-
- public String getCommitTableName() {
- return commitTableName;
- }
-
- @Inject(optional = true)
- @Named("omid.client.hbase.commitTableName")
- public void setCommitTableName(String commitTableName) {
- this.commitTableName = commitTableName;
- }
-
- public OmidClientConfiguration getOmidClientConfiguration() {
- return omidClientConfiguration;
- }
-
- public void setOmidClientConfiguration(OmidClientConfiguration omidClientConfiguration) {
- this.omidClientConfiguration = omidClientConfiguration;
- }
-
- public MetricsRegistry getMetrics() {
- return metrics;
- }
-
- @Inject(optional = true)
- @Named("omid.client.hbase.metrics")
- public void setMetrics(MetricsRegistry metrics) {
- this.metrics = metrics;
- }
-
- // Delegation to make end-user life better
-
- public OmidClientConfiguration.ConnType getConnectionType() {
- return omidClientConfiguration.getConnectionType();
- }
-
- public void setReconnectionDelayInSecs(int reconnectionDelayInSecs) {
- omidClientConfiguration.setReconnectionDelayInSecs(reconnectionDelayInSecs);
- }
-
- public void setExecutorThreads(int executorThreads) {
- omidClientConfiguration.setExecutorThreads(executorThreads);
- }
-
- public int getRequestTimeoutInMs() {
- return omidClientConfiguration.getRequestTimeoutInMs();
- }
-
- public void setConnectionString(String connectionString) {
- omidClientConfiguration.setConnectionString(connectionString);
- }
-
- public void setRequestTimeoutInMs(int requestTimeoutInMs) {
- omidClientConfiguration.setRequestTimeoutInMs(requestTimeoutInMs);
- }
-
- public void setZkConnectionTimeoutInSecs(int zkConnectionTimeoutInSecs) {
- omidClientConfiguration.setZkConnectionTimeoutInSecs(zkConnectionTimeoutInSecs);
- }
-
- public void setConnectionType(OmidClientConfiguration.ConnType connectionType) {
- omidClientConfiguration.setConnectionType(connectionType);
- }
-
- public void setRequestMaxRetries(int requestMaxRetries) {
- omidClientConfiguration.setRequestMaxRetries(requestMaxRetries);
- }
-
- public int getZkConnectionTimeoutInSecs() {
- return omidClientConfiguration.getZkConnectionTimeoutInSecs();
- }
-
- public void setRetryDelayInMs(int retryDelayInMs) {
- omidClientConfiguration.setRetryDelayInMs(retryDelayInMs);
- }
-
- public int getExecutorThreads() {
- return omidClientConfiguration.getExecutorThreads();
- }
-
- public int getRetryDelayInMs() {
- return omidClientConfiguration.getRetryDelayInMs();
- }
-
- public String getConnectionString() {
- return omidClientConfiguration.getConnectionString();
- }
-
- public int getRequestMaxRetries() {
- return omidClientConfiguration.getRequestMaxRetries();
- }
-
- public int getReconnectionDelayInSecs() {
- return omidClientConfiguration.getReconnectionDelayInSecs();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
deleted file mode 100644
index 0fc3cd1..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseSyncPostCommitter.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.MetricsRegistry;
-import com.yahoo.omid.metrics.Timer;
-import com.yahoo.omid.tso.client.CellId;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.ExecutionException;
-
-import static com.yahoo.omid.metrics.MetricsUtils.name;
-
-public class HBaseSyncPostCommitter implements PostCommitActions {
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseSyncPostCommitter.class);
-
- private final MetricsRegistry metrics;
- private final CommitTable.Client commitTableClient;
-
- private final Timer commitTableUpdateTimer;
- private final Timer shadowCellsUpdateTimer;
-
- public HBaseSyncPostCommitter(MetricsRegistry metrics, CommitTable.Client commitTableClient) {
- this.metrics = metrics;
- this.commitTableClient = commitTableClient;
-
- this.commitTableUpdateTimer = metrics.timer(name("omid", "tm", "hbase", "commitTableUpdate", "latency"));
- this.shadowCellsUpdateTimer = metrics.timer(name("omid", "tm", "hbase", "shadowCellsUpdate", "latency"));
- }
-
- @Override
- public ListenableFuture<Void> updateShadowCells(AbstractTransaction<? extends CellId> transaction) {
-
- SettableFuture<Void> updateSCFuture = SettableFuture.create();
-
- HBaseTransaction tx = HBaseTransactionManager.enforceHBaseTransactionAsParam(transaction);
-
- shadowCellsUpdateTimer.start();
- try {
-
- // Add shadow cells
- for (HBaseCellId cell : tx.getWriteSet()) {
- Put put = new Put(cell.getRow());
- put.add(cell.getFamily(),
- CellUtils.addShadowCellSuffix(cell.getQualifier(), 0, cell.getQualifier().length),
- tx.getStartTimestamp(),
- Bytes.toBytes(tx.getCommitTimestamp()));
- try {
- cell.getTable().put(put);
- } catch (IOException e) {
- LOG.warn("{}: Error inserting shadow cell {}", tx, cell, e);
- updateSCFuture.setException(
- new TransactionManagerException(tx + ": Error inserting shadow cell " + cell, e));
- }
- }
-
- // Flush affected tables before returning to avoid loss of shadow cells updates when autoflush is disabled
- try {
- tx.flushTables();
- updateSCFuture.set(null);
- } catch (IOException e) {
- LOG.warn("{}: Error while flushing writes", tx, e);
- updateSCFuture.setException(new TransactionManagerException(tx + ": Error while flushing writes", e));
- }
-
- } finally {
- shadowCellsUpdateTimer.stop();
- }
-
- return updateSCFuture;
-
- }
-
- @Override
- public ListenableFuture<Void> removeCommitTableEntry(AbstractTransaction<? extends CellId> transaction) {
-
- SettableFuture<Void> updateSCFuture = SettableFuture.create();
-
- HBaseTransaction tx = HBaseTransactionManager.enforceHBaseTransactionAsParam(transaction);
-
- commitTableUpdateTimer.start();
-
- try {
- commitTableClient.completeTransaction(tx.getStartTimestamp()).get();
- updateSCFuture.set(null);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.warn("{}: interrupted during commit table entry delete", tx, e);
- updateSCFuture.setException(
- new TransactionManagerException(tx + ": interrupted during commit table entry delete"));
- } catch (ExecutionException e) {
- LOG.warn("{}: can't remove commit table entry", tx, e);
- updateSCFuture.setException(new TransactionManagerException(tx + ": can't remove commit table entry"));
- } finally {
- commitTableUpdateTimer.stop();
- }
-
- return updateSCFuture;
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
deleted file mode 100644
index 4acda7d..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransaction.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-public class HBaseTransaction extends AbstractTransaction<HBaseCellId> {
- private static final Logger LOG = LoggerFactory.getLogger(HBaseTransaction.class);
-
- HBaseTransaction(long transactionId, long epoch, Set<HBaseCellId> writeSet, AbstractTransactionManager tm) {
- super(transactionId, epoch, writeSet, tm);
- }
-
- @Override
- public void cleanup() {
- Set<HBaseCellId> writeSet = getWriteSet();
- for (final HBaseCellId cell : writeSet) {
- Delete delete = new Delete(cell.getRow());
- delete.deleteColumn(cell.getFamily(), cell.getQualifier(), getStartTimestamp());
- try {
- cell.getTable().delete(delete);
- } catch (IOException e) {
- LOG.warn("Failed cleanup cell {} for Tx {}. This issue has been ignored", new Object[]{cell, getTransactionId(), e});
- }
- }
- try {
- flushTables();
- } catch (IOException e) {
- LOG.warn("Failed flushing tables for Tx {}", getTransactionId(), e);
- }
- }
-
- /**
- * Flushes pending operations for tables touched by transaction
- */
- public void flushTables() throws IOException {
-
- for (HTableInterface writtenTable : getWrittenTables()) {
- writtenTable.flushCommits();
- }
-
- }
-
- // ****************************************************************************************************************
- // Helper methods
- // ****************************************************************************************************************
-
- private Set<HTableInterface> getWrittenTables() {
- HashSet<HBaseCellId> writeSet = (HashSet<HBaseCellId>) getWriteSet();
- Set<HTableInterface> tables = new HashSet<HTableInterface>();
- for (HBaseCellId cell : writeSet) {
- tables.add(cell.getTable());
- }
- return tables;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
deleted file mode 100644
index dd33832..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionClient.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-public interface HBaseTransactionClient {
- boolean isCommitted(HBaseCellId hBaseCellId) throws TransactionException;
-
- long getLowWatermark() throws TransactionException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java b/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
deleted file mode 100644
index 941c583..0000000
--- a/hbase-client/src/main/java/com/yahoo/omid/transaction/HBaseTransactionManager.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.CommitTable.CommitTimestamp;
-import com.yahoo.omid.committable.hbase.HBaseCommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.tools.hbase.HBaseLogin;
-import com.yahoo.omid.tso.client.CellId;
-import com.yahoo.omid.tso.client.TSOClient;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-
-public class HBaseTransactionManager extends AbstractTransactionManager implements HBaseTransactionClient {
-
- private static final Logger LOG = LoggerFactory.getLogger(HBaseTransactionManager.class);
-
- private static class HBaseTransactionFactory implements TransactionFactory<HBaseCellId> {
-
- @Override
- public HBaseTransaction createTransaction(long transactionId, long epoch, AbstractTransactionManager tm) {
-
- return new HBaseTransaction(transactionId, epoch, new HashSet<HBaseCellId>(), tm);
-
- }
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Construction
- // ----------------------------------------------------------------------------------------------------------------
-
- public static TransactionManager newInstance() throws IOException, InterruptedException {
- return newInstance(new HBaseOmidClientConfiguration());
- }
-
- public static TransactionManager newInstance(HBaseOmidClientConfiguration configuration)
- throws IOException, InterruptedException {
- //Logging in to Secure HBase if required
- HBaseLogin.loginIfNeeded(configuration);
- return builder(configuration).build();
- }
-
- @VisibleForTesting
- static class Builder {
-
- // Required parameters
- private final HBaseOmidClientConfiguration hbaseOmidClientConf;
-
- // Optional parameters - initialized to default values
- private Optional<TSOClient> tsoClient = Optional.absent();
- private Optional<CommitTable.Client> commitTableClient = Optional.absent();
- private Optional<PostCommitActions> postCommitter = Optional.absent();
-
- private Builder(HBaseOmidClientConfiguration hbaseOmidClientConf) {
- this.hbaseOmidClientConf = hbaseOmidClientConf;
- }
-
- Builder tsoClient(TSOClient tsoClient) {
- this.tsoClient = Optional.of(tsoClient);
- return this;
- }
-
- Builder commitTableClient(CommitTable.Client client) {
- this.commitTableClient = Optional.of(client);
- return this;
- }
-
- Builder postCommitter(PostCommitActions postCommitter) {
- this.postCommitter = Optional.of(postCommitter);
- return this;
- }
-
- HBaseTransactionManager build() throws IOException, InterruptedException {
-
- CommitTable.Client commitTableClient = this.commitTableClient.or(buildCommitTableClient()).get();
- PostCommitActions postCommitter = this.postCommitter.or(buildPostCommitter(commitTableClient)).get();
- TSOClient tsoClient = this.tsoClient.or(buildTSOClient()).get();
-
- return new HBaseTransactionManager(hbaseOmidClientConf,
- postCommitter,
- tsoClient,
- commitTableClient,
- new HBaseTransactionFactory());
- }
-
- private Optional<TSOClient> buildTSOClient() throws IOException, InterruptedException {
- return Optional.of(TSOClient.newInstance(hbaseOmidClientConf.getOmidClientConfiguration()));
- }
-
-
- private Optional<CommitTable.Client> buildCommitTableClient() throws IOException {
- HBaseCommitTableConfig commitTableConf = new HBaseCommitTableConfig();
- commitTableConf.setTableName(hbaseOmidClientConf.getCommitTableName());
- CommitTable commitTable = new HBaseCommitTable(hbaseOmidClientConf.getHBaseConfiguration(), commitTableConf);
- return Optional.of(commitTable.getClient());
- }
-
- private Optional<PostCommitActions> buildPostCommitter(CommitTable.Client commitTableClient ) {
-
- PostCommitActions postCommitter;
- PostCommitActions syncPostCommitter = new HBaseSyncPostCommitter(hbaseOmidClientConf.getMetrics(),
- commitTableClient);
- switch(hbaseOmidClientConf.getPostCommitMode()) {
- case ASYNC:
- ListeningExecutorService postCommitExecutor =
- MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
- postCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
- break;
- case SYNC:
- default:
- postCommitter = syncPostCommitter;
- break;
- }
-
- return Optional.of(postCommitter);
- }
-
- }
-
- @VisibleForTesting
- static Builder builder(HBaseOmidClientConfiguration hbaseOmidClientConf) {
- return new Builder(hbaseOmidClientConf);
- }
-
- private HBaseTransactionManager(HBaseOmidClientConfiguration hBaseOmidClientConfiguration,
- PostCommitActions postCommitter,
- TSOClient tsoClient,
- CommitTable.Client commitTableClient,
- HBaseTransactionFactory hBaseTransactionFactory) {
-
- super(hBaseOmidClientConfiguration.getMetrics(),
- postCommitter,
- tsoClient,
- commitTableClient,
- hBaseTransactionFactory);
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // AbstractTransactionManager overwritten methods
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public void preCommit(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {
- try {
- // Flush all pending writes
- HBaseTransaction hBaseTx = enforceHBaseTransactionAsParam(transaction);
- hBaseTx.flushTables();
- } catch (IOException e) {
- throw new TransactionManagerException("Exception while flushing writes", e);
- }
- }
-
- @Override
- public void preRollback(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {
- try {
- // Flush all pending writes
- HBaseTransaction hBaseTx = enforceHBaseTransactionAsParam(transaction);
- hBaseTx.flushTables();
- } catch (IOException e) {
- throw new TransactionManagerException("Exception while flushing writes", e);
- }
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // HBaseTransactionClient method implementations
- // ----------------------------------------------------------------------------------------------------------------
-
- @Override
- public boolean isCommitted(HBaseCellId hBaseCellId) throws TransactionException {
- try {
- CommitTimestamp tentativeCommitTimestamp =
- locateCellCommitTimestamp(hBaseCellId.getTimestamp(), tsoClient.getEpoch(),
- new CommitTimestampLocatorImpl(hBaseCellId, Maps.<Long, Long>newHashMap()));
-
- // If transaction that added the cell was invalidated
- if (!tentativeCommitTimestamp.isValid()) {
- return false;
- }
-
- switch (tentativeCommitTimestamp.getLocation()) {
- case COMMIT_TABLE:
- case SHADOW_CELL:
- return true;
- case NOT_PRESENT:
- return false;
- case CACHE: // cache was empty
- default:
- return false;
- }
- } catch (IOException e) {
- throw new TransactionException("Failure while checking if a transaction was committed", e);
- }
- }
-
- @Override
- public long getLowWatermark() throws TransactionException {
- try {
- return commitTableClient.readLowWatermark().get();
- } catch (ExecutionException ee) {
- throw new TransactionException("Error reading low watermark", ee.getCause());
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- throw new TransactionException("Interrupted reading low watermark", ie);
- }
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods
- // ----------------------------------------------------------------------------------------------------------------
-
- static HBaseTransaction enforceHBaseTransactionAsParam(AbstractTransaction<? extends CellId> tx) {
-
- if (tx instanceof HBaseTransaction) {
- return (HBaseTransaction) tx;
- } else {
- throw new IllegalArgumentException(
- "The transaction object passed is not an instance of HBaseTransaction");
- }
-
- }
-
- static class CommitTimestampLocatorImpl implements CommitTimestampLocator {
-
- private HBaseCellId hBaseCellId;
- private final Map<Long, Long> commitCache;
-
- CommitTimestampLocatorImpl(HBaseCellId hBaseCellId, Map<Long, Long> commitCache) {
- this.hBaseCellId = hBaseCellId;
- this.commitCache = commitCache;
- }
-
- @Override
- public Optional<Long> readCommitTimestampFromCache(long startTimestamp) {
- if (commitCache.containsKey(startTimestamp)) {
- return Optional.of(commitCache.get(startTimestamp));
- }
- return Optional.absent();
- }
-
- @Override
- public Optional<Long> readCommitTimestampFromShadowCell(long startTimestamp) throws IOException {
-
- Get get = new Get(hBaseCellId.getRow());
- byte[] family = hBaseCellId.getFamily();
- byte[] shadowCellQualifier = CellUtils.addShadowCellSuffix(hBaseCellId.getQualifier());
- get.addColumn(family, shadowCellQualifier);
- get.setMaxVersions(1);
- get.setTimeStamp(startTimestamp);
- Result result = hBaseCellId.getTable().get(get);
- if (result.containsColumn(family, shadowCellQualifier)) {
- return Optional.of(Bytes.toLong(result.getValue(family, shadowCellQualifier)));
- }
- return Optional.absent();
- }
-
- }
-
-}
[36/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
deleted file mode 100644
index 8394a01..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestUpdateScan.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.testng.AssertJUnit.assertEquals;
-import static org.testng.AssertJUnit.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestUpdateScan extends OmidTestBase {
- private static final Logger LOG = LoggerFactory.getLogger(TestUpdateScan.class);
-
- private static final String TEST_COL = "value";
- private static final String TEST_COL_2 = "col_2";
-
- @Test
- public void testGet(ITestContext context) throws Exception {
- try {
- TransactionManager tm = newTransactionManager(context);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
- Transaction t = tm.begin();
- int[] lInts = new int[]{100, 243, 2342, 22, 1, 5, 43, 56};
- for (int i = 0; i < lInts.length; i++) {
- byte[] data = Bytes.toBytes(lInts[i]);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
- table.put(t, put);
- }
- int startKeyValue = lInts[3];
- int stopKeyValue = lInts[3];
- byte[] startKey = Bytes.toBytes(startKeyValue);
- byte[] stopKey = Bytes.toBytes(stopKeyValue);
- Get g = new Get(startKey);
- Result r = table.get(t, g);
- if (!r.isEmpty()) {
- int tmp = Bytes.toInt(r.getValue(Bytes.toBytes(TEST_FAMILY),
- Bytes.toBytes(TEST_COL)));
- LOG.info("Result:" + tmp);
- assertTrue("Bad value, should be "
- + startKeyValue + " but is " + tmp
- , tmp == startKeyValue);
- } else {
- Assert.fail("Bad result");
- }
- tm.commit(t);
-
- Scan s = new Scan(startKey);
- CompareFilter.CompareOp op = CompareFilter.CompareOp.LESS_OR_EQUAL;
- RowFilter toFilter = new RowFilter(op, new BinaryPrefixComparator(stopKey));
- boolean startInclusive = true;
- if (!startInclusive) {
- FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
- filters.addFilter(new RowFilter(CompareFilter.CompareOp.GREATER,
- new BinaryPrefixComparator(startKey)));
- filters.addFilter(new WhileMatchFilter(toFilter));
- s.setFilter(filters);
- } else {
- s.setFilter(new WhileMatchFilter(toFilter));
- }
- t = tm.begin();
- ResultScanner res = table.getScanner(t, s);
- Result rr;
- int count = 0;
- while ((rr = res.next()) != null) {
- int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
- Bytes.toBytes(TEST_COL)));
- LOG.info("Result: " + iTmp);
- count++;
- }
- assertEquals("Count is wrong", 1, count);
- LOG.info("Rows found " + count);
- tm.commit(t);
- table.close();
- } catch (Exception e) {
- LOG.error("Exception in test", e);
- }
- }
-
- @Test
- public void testScan(ITestContext context) throws Exception {
-
- try (TTable table = new TTable(hbaseConf, TEST_TABLE)) {
- TransactionManager tm = newTransactionManager(context);
- Transaction t = tm.begin();
- int[] lInts = new int[]{100, 243, 2342, 22, 1, 5, 43, 56};
- for (int lInt : lInts) {
- byte[] data = Bytes.toBytes(lInt);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL_2), data);
- table.put(t, put);
- }
-
- Scan s = new Scan();
- // Adding two columns to the scanner should not throw a
- // ConcurrentModificationException when getting the scanner
- s.addColumn(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL));
- s.addColumn(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL_2));
- ResultScanner res = table.getScanner(t, s);
- Result rr;
- int count = 0;
- while ((rr = res.next()) != null) {
- int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
- Bytes.toBytes(TEST_COL)));
- LOG.info("Result: " + iTmp);
- count++;
- }
- assertTrue("Count should be " + lInts.length + " but is " + count,
- count == lInts.length);
- LOG.info("Rows found " + count);
-
- tm.commit(t);
-
- t = tm.begin();
- res = table.getScanner(t, s);
- count = 0;
- while ((rr = res.next()) != null) {
- int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
- Bytes.toBytes(TEST_COL)));
- LOG.info("Result: " + iTmp);
- count++;
- }
- assertTrue("Count should be " + lInts.length + " but is " + count,
- count == lInts.length);
- LOG.info("Rows found " + count);
- tm.commit(t);
- }
-
- }
-
-
- @Test
- public void testScanUncommitted(ITestContext context) throws Exception {
- try {
- TransactionManager tm = newTransactionManager(context);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
- Transaction t = tm.begin();
- int[] lIntsA = new int[]{100, 243, 2342, 22, 1, 5, 43, 56};
- for (int aLIntsA : lIntsA) {
- byte[] data = Bytes.toBytes(aLIntsA);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
- table.put(t, put);
- }
- tm.commit(t);
-
- Transaction tu = tm.begin();
- int[] lIntsB = new int[]{105, 24, 4342, 32, 7, 3, 30, 40};
- for (int aLIntsB : lIntsB) {
- byte[] data = Bytes.toBytes(aLIntsB);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
- table.put(tu, put);
- }
-
- t = tm.begin();
- int[] lIntsC = new int[]{109, 224, 242, 2, 16, 59, 23, 26};
- for (int aLIntsC : lIntsC) {
- byte[] data = Bytes.toBytes(aLIntsC);
- Put put = new Put(data);
- put.add(Bytes.toBytes(TEST_FAMILY), Bytes.toBytes(TEST_COL), data);
- table.put(t, put);
- }
- tm.commit(t);
-
- t = tm.begin();
- Scan s = new Scan();
- ResultScanner res = table.getScanner(t, s);
- Result rr;
- int count = 0;
-
- while ((rr = res.next()) != null) {
- int iTmp = Bytes.toInt(rr.getValue(Bytes.toBytes(TEST_FAMILY),
- Bytes.toBytes(TEST_COL)));
- LOG.info("Result: " + iTmp);
- count++;
- }
- assertTrue("Count should be " + (lIntsA.length * lIntsC.length) + " but is " + count,
- count == lIntsA.length + lIntsC.length);
- LOG.info("Rows found " + count);
- tm.commit(t);
- table.close();
- } catch (Exception e) {
- LOG.error("Exception in test", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/OmidTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/OmidTestBase.java b/hbase-client/src/test/java/org/apache/omid/transaction/OmidTestBase.java
new file mode 100644
index 0000000..e4c4614
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/OmidTestBase.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.apache.omid.TestUtils;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.InMemoryCommitTable;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
+import org.apache.omid.tools.hbase.OmidTableManager;
+import org.apache.omid.tso.TSOMockModule;
+import org.apache.omid.tso.TSOServer;
+import org.apache.omid.tso.TSOServerConfig;
+import org.apache.omid.tso.client.OmidClientConfiguration;
+import org.apache.omid.tso.client.TSOClient;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.AfterGroups;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeGroups;
+import org.testng.annotations.BeforeMethod;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+
+import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
+
+public abstract class OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(OmidTestBase.class);
+
+ static HBaseTestingUtility hBaseUtils;
+ private static MiniHBaseCluster hbaseCluster;
+ static Configuration hbaseConf;
+
+ protected static final String TEST_TABLE = "test";
+ protected static final String TEST_FAMILY = "data";
+ static final String TEST_FAMILY2 = "data2";
+ private HBaseCommitTableConfig hBaseCommitTableConfig;
+
+ @BeforeMethod(alwaysRun = true)
+ public void beforeClass(Method method) throws Exception {
+ Thread.currentThread().setName("UnitTest-" + method.getName());
+ }
+
+
+ @BeforeGroups(groups = "sharedHBase")
+ public void beforeGroups(ITestContext context) throws Exception {
+ // TSO Setup
+ TSOServerConfig tsoConfig = new TSOServerConfig();
+ tsoConfig.setPort(1234);
+ tsoConfig.setMaxItems(1000);
+ Injector injector = Guice.createInjector(new TSOMockModule(tsoConfig));
+ LOG.info("Starting TSO");
+ TSOServer tso = injector.getInstance(TSOServer.class);
+ hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);
+ HBaseTimestampStorageConfig hBaseTimestampStorageConfig = injector.getInstance(HBaseTimestampStorageConfig.class);
+ tso.startAndWait();
+ TestUtils.waitForSocketListening("localhost", 1234, 100);
+ LOG.info("Finished loading TSO");
+ context.setAttribute("tso", tso);
+
+ OmidClientConfiguration clientConf = new OmidClientConfiguration();
+ clientConf.setConnectionString("localhost:1234");
+ context.setAttribute("clientConf", clientConf);
+
+ InMemoryCommitTable commitTable = (InMemoryCommitTable) injector.getInstance(CommitTable.class);
+ context.setAttribute("commitTable", commitTable);
+
+ // Create the associated Handler
+ TSOClient client = TSOClient.newInstance(clientConf);
+ context.setAttribute("client", client);
+
+ // ------------------------------------------------------------------------------------------------------------
+ // HBase setup
+ // ------------------------------------------------------------------------------------------------------------
+ LOG.info("Creating HBase minicluster");
+ hbaseConf = HBaseConfiguration.create();
+ hbaseConf.setInt("hbase.hregion.memstore.flush.size", 10_000 * 1024);
+ hbaseConf.setInt("hbase.regionserver.nbreservationblocks", 1);
+ hbaseConf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 3);
+
+ File tempFile = File.createTempFile("OmidTest", "");
+ tempFile.deleteOnExit();
+ hbaseConf.set("hbase.rootdir", tempFile.getAbsolutePath());
+
+ hBaseUtils = new HBaseTestingUtility(hbaseConf);
+ hbaseCluster = hBaseUtils.startMiniCluster(1);
+ hBaseUtils.createTable(Bytes.toBytes(hBaseTimestampStorageConfig.getTableName()),
+ new byte[][]{hBaseTimestampStorageConfig.getFamilyName().getBytes()},
+ Integer.MAX_VALUE);
+
+ createTestTable();
+ createCommitTable();
+
+ LOG.info("HBase minicluster is up");
+ }
+
+ private void createTestTable() throws IOException {
+ HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
+ HTableDescriptor test_table_desc = new HTableDescriptor(TableName.valueOf(TEST_TABLE));
+ HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY);
+ HColumnDescriptor datafam2 = new HColumnDescriptor(TEST_FAMILY2);
+ datafam.setMaxVersions(Integer.MAX_VALUE);
+ datafam2.setMaxVersions(Integer.MAX_VALUE);
+ test_table_desc.addFamily(datafam);
+ test_table_desc.addFamily(datafam2);
+ admin.createTable(test_table_desc);
+ }
+
+ private void createCommitTable() throws IOException {
+ String[] args = new String[]{OmidTableManager.COMMIT_TABLE_COMMAND_NAME, "-numRegions", "1"};
+ OmidTableManager omidTableManager = new OmidTableManager(args);
+ omidTableManager.executeActionsOnHBase(hbaseConf);
+ }
+
+
+ private TSOServer getTSO(ITestContext context) {
+ return (TSOServer) context.getAttribute("tso");
+ }
+
+
+ TSOClient getClient(ITestContext context) {
+ return (TSOClient) context.getAttribute("client");
+ }
+
+ InMemoryCommitTable getCommitTable(ITestContext context) {
+ return (InMemoryCommitTable) context.getAttribute("commitTable");
+ }
+
+ protected TransactionManager newTransactionManager(ITestContext context) throws Exception {
+ return newTransactionManager(context, getClient(context));
+ }
+
+ protected TransactionManager newTransactionManager(ITestContext context, PostCommitActions postCommitActions) throws Exception {
+ HBaseOmidClientConfiguration clientConf = new HBaseOmidClientConfiguration();
+ clientConf.setConnectionString("localhost:1234");
+ clientConf.setHBaseConfiguration(hbaseConf);
+ return HBaseTransactionManager.builder(clientConf)
+ .postCommitter(postCommitActions)
+ .commitTableClient(getCommitTable(context).getClient())
+ .tsoClient(getClient(context)).build();
+ }
+
+ protected TransactionManager newTransactionManager(ITestContext context, TSOClient tsoClient) throws Exception {
+ HBaseOmidClientConfiguration clientConf = new HBaseOmidClientConfiguration();
+ clientConf.setConnectionString("localhost:1234");
+ clientConf.setHBaseConfiguration(hbaseConf);
+ return HBaseTransactionManager.builder(clientConf)
+ .commitTableClient(getCommitTable(context).getClient())
+ .tsoClient(tsoClient).build();
+ }
+
+ protected TransactionManager newTransactionManager(ITestContext context, CommitTable.Client commitTableClient)
+ throws Exception {
+ HBaseOmidClientConfiguration clientConf = new HBaseOmidClientConfiguration();
+ clientConf.setConnectionString("localhost:1234");
+ clientConf.setHBaseConfiguration(hbaseConf);
+ return HBaseTransactionManager.builder(clientConf)
+ .commitTableClient(commitTableClient)
+ .tsoClient(getClient(context)).build();
+ }
+
+ @AfterGroups(groups = "sharedHBase")
+ public void afterGroups(ITestContext context) throws Exception {
+ LOG.info("Tearing down OmidTestBase...");
+ if (hbaseCluster != null) {
+ hBaseUtils.shutdownMiniCluster();
+ }
+
+ getClient(context).close().get();
+ getTSO(context).stopAndWait();
+ TestUtils.waitForSocketNotListening("localhost", 1234, 1000);
+ }
+
+ @AfterMethod(groups = "sharedHBase", timeOut = 60_000)
+ public void afterMethod() {
+ try {
+ LOG.info("tearing Down");
+ HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
+ deleteTable(admin, TableName.valueOf(TEST_TABLE));
+ createTestTable();
+ deleteTable(admin, TableName.valueOf(hBaseCommitTableConfig.getTableName()));
+ createCommitTable();
+ } catch (Exception e) {
+ LOG.error("Error tearing down", e);
+ }
+ }
+
+ void deleteTable(HBaseAdmin admin, TableName tableName) throws IOException {
+ if (admin.tableExists(tableName)) {
+ if (admin.isTableDisabled(tableName)) {
+ admin.deleteTable(tableName);
+ } else {
+ admin.disableTable(tableName);
+ admin.deleteTable(tableName);
+ }
+ }
+ }
+
+ static boolean verifyValue(byte[] tableName, byte[] row,
+ byte[] fam, byte[] col, byte[] value) {
+
+ try (HTable table = new HTable(hbaseConf, tableName)) {
+ Get g = new Get(row).setMaxVersions(1);
+ Result r = table.get(g);
+ Cell cell = r.getColumnLatestCell(fam, col);
+
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Value for " + Bytes.toString(tableName) + ":"
+ + Bytes.toString(row) + ":" + Bytes.toString(fam)
+ + Bytes.toString(col) + "=>" + Bytes.toString(CellUtil.cloneValue(cell))
+ + " (" + Bytes.toString(value) + " expected)");
+ }
+
+ return Bytes.equals(CellUtil.cloneValue(cell), value);
+ } catch (IOException e) {
+ LOG.error("Error reading row " + Bytes.toString(tableName) + ":"
+ + Bytes.toString(row) + ":" + Bytes.toString(fam)
+ + Bytes.toString(col), e);
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestAsynchronousPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestAsynchronousPostCommitter.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestAsynchronousPostCommitter.java
new file mode 100644
index 0000000..21abacf
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestAsynchronousPostCommitter.java
@@ -0,0 +1,323 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.metrics.NullMetricsProvider;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestAsynchronousPostCommitter extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestAsynchronousPostCommitter.class);
+
+ private static final byte[] family = Bytes.toBytes(TEST_FAMILY);
+ private static final byte[] nonExistentFamily = Bytes.toBytes("non-existent");
+ private static final byte[] qualifier = Bytes.toBytes("test-qual");
+
+ byte[] row1 = Bytes.toBytes("test-is-committed1");
+ byte[] row2 = Bytes.toBytes("test-is-committed2");
+
+ @Test(timeOut = 30_000)
+ public void testPostCommitActionsAreCalledAsynchronously(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = getCommitTable(context).getClient();
+
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ ListeningExecutorService postCommitExecutor =
+ MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
+ PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
+
+ TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
+
+ final CountDownLatch beforeUpdatingShadowCellsLatch = new CountDownLatch(1);
+ final CountDownLatch afterUpdatingShadowCellsLatch = new CountDownLatch(1);
+ final CountDownLatch beforeRemovingCTEntryLatch = new CountDownLatch(1);
+ final CountDownLatch afterRemovingCTEntryLatch = new CountDownLatch(1);
+
+ doAnswer(new Answer<ListenableFuture<Void>>() {
+ public ListenableFuture<Void> answer(InvocationOnMock invocation) {
+ try {
+ beforeUpdatingShadowCellsLatch.await();
+ invocation.callRealMethod();
+ afterUpdatingShadowCellsLatch.countDown();
+ } catch (Throwable throwable) {
+ throwable.printStackTrace();
+ }
+ return SettableFuture.create();
+ }
+ }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
+
+ doAnswer(new Answer<ListenableFuture<Void>>() {
+ public ListenableFuture<Void> answer(InvocationOnMock invocation) {
+ try {
+ beforeRemovingCTEntryLatch.await();
+ LOG.info("We are here");
+ invocation.callRealMethod();
+ afterRemovingCTEntryLatch.countDown();
+ } catch (Throwable throwable) {
+ throwable.printStackTrace();
+ }
+ return SettableFuture.create();
+ }
+ }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Execute tx with async post commit actions
+ Transaction tx1 = tm.begin();
+
+ Put put1 = new Put(row1);
+ put1.add(family, qualifier, Bytes.toBytes("hey!"));
+ txTable.put(tx1, put1);
+ Put put2 = new Put(row2);
+ put2.add(family, qualifier, Bytes.toBytes("hou!"));
+ txTable.put(tx1, put2);
+
+ tm.commit(tx1);
+
+ long tx1Id = tx1.getTransactionId();
+
+ // As we have paused the update of shadow cells, the shadow cells shouldn't be there yet
+ assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+
+ // Commit Table should contain an entry for the transaction
+ Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+ assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
+
+ // Read from row1 and row2 in a different Tx and check that result is the data written by tx1 despite the
+ // post commit actions have not been executed yet (the shadow cells healing process should make its work)
+ Transaction tx2 = tm.begin();
+ Get get1 = new Get(row1);
+ Result result = txTable.get(tx2, get1);
+ byte[] value = result.getValue(family, qualifier);
+ assertNotNull(value);
+ assertEquals("hey!", Bytes.toString(value));
+
+ Get get2 = new Get(row2);
+ result = txTable.get(tx2, get2);
+ value = result.getValue(family, qualifier);
+ assertNotNull(value);
+ assertEquals("hou!", Bytes.toString(value));
+
+ // Then, we continue with the update of shadow cells and we wait till completed
+ beforeUpdatingShadowCellsLatch.countDown();
+ afterUpdatingShadowCellsLatch.await();
+
+ // Now we can check that the shadow cells are there...
+ verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
+ assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ // ...and the transaction entry is still in the Commit Table
+ commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+ assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
+
+ // Finally, we continue till the Commit Table cleaning process is done...
+ beforeRemovingCTEntryLatch.countDown();
+ afterRemovingCTEntryLatch.await();
+
+ // ...so now, the Commit Table should NOT contain the entry for the transaction anymore
+ verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
+ commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
+ assertFalse(commitTimestamp.isPresent());
+
+ // Final checks
+ verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
+ verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
+
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testNoAsyncPostActionsAreCalled(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = getCommitTable(context).getClient();
+
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ ListeningExecutorService postCommitExecutor =
+ MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
+ PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
+
+ TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
+
+ final CountDownLatch updateShadowCellsCalledLatch = new CountDownLatch(1);
+ final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
+
+ // Simulate shadow cells are not updated and commit table is not clean
+ doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) {
+ // Do not invoke real method simulating a fail of the shadow cells update
+ updateShadowCellsCalledLatch.countDown();
+ return null;
+ }
+ }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
+
+ doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) {
+ // Do not invoke real method simulating a fail of the async clean of commit table entry
+ removeCommitTableEntryCalledLatch.countDown();
+ return null;
+ }
+ }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
+
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Execute tx with async post commit actions
+ Transaction tx1 = tm.begin();
+
+ Put put1 = new Put(row1);
+ put1.add(family, qualifier, Bytes.toBytes("hey!"));
+ txTable.put(tx1, put1);
+ Put put2 = new Put(row2);
+ put2.add(family, qualifier, Bytes.toBytes("hou!"));
+ txTable.put(tx1, put2);
+
+ tm.commit(tx1);
+
+ long tx1Id = tx1.getTransactionId();
+
+ // The shadow cells shouldn't be there...
+ assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ // ... and the should NOT have been cleaned
+ Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+
+ updateShadowCellsCalledLatch.await();
+
+ // Not even after waiting for the method call on the shadow cells update...
+ assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+
+ removeCommitTableEntryCalledLatch.await();
+ // ... and after waiting for the method call for cleaning the commit table entry
+ commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+
+ // Final checks
+ verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
+ verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
+
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testOnlyShadowCellsUpdateIsExecuted(ITestContext context) throws Exception {
+
+ CommitTable.Client commitTableClient = getCommitTable(context).getClient();
+
+ PostCommitActions syncPostCommitter =
+ spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
+ ListeningExecutorService postCommitExecutor =
+ MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
+ PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
+
+ TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
+
+ final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
+
+ doAnswer(new Answer<Void>() {
+ public Void answer(InvocationOnMock invocation) {
+ // Do not invoke real method simulating a fail of the async clean of commit table entry
+ removeCommitTableEntryCalledLatch.countDown();
+ return null;
+ }
+ }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
+
+
+ try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
+
+ // Execute tx with async post commit actions
+ Transaction tx1 = tm.begin();
+
+ Put put1 = new Put(row1);
+ put1.add(family, qualifier, Bytes.toBytes("hey!"));
+ txTable.put(tx1, put1);
+ Put put2 = new Put(row2);
+ put2.add(family, qualifier, Bytes.toBytes("hou!"));
+ txTable.put(tx1, put2);
+
+ tm.commit(tx1);
+
+ long tx1Id = tx1.getTransactionId();
+
+ // We continue when the unsuccessful call of the method for cleaning commit table has been invoked
+ removeCommitTableEntryCalledLatch.await();
+
+ // We check that the shadow cells are there (because the update of the shadow cells should precede
+ // the cleaning of the commit table entry) ...
+ assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+ assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
+
+ // ... and the commit table entry has NOT been cleaned
+ Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
+ assertTrue(commitTimestamp.isPresent());
+ assertTrue(commitTimestamp.get().isValid());
+
+ // Final checks
+ verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
+ verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestAutoFlush.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestAutoFlush.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestAutoFlush.java
new file mode 100644
index 0000000..297dd01
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestAutoFlush.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
+
+@Test(groups = "sharedHBase")
+public class TestAutoFlush extends OmidTestBase {
+
+ @Test
+ public void testReadWithSeveralUncommitted(ITestContext context) throws Exception {
+ byte[] family = Bytes.toBytes(TEST_FAMILY);
+ byte[] row = Bytes.toBytes("row");
+ byte[] col = Bytes.toBytes("col1");
+ byte[] data = Bytes.toBytes("data");
+ TransactionManager tm = newTransactionManager(context);
+ TTable table = new TTable(hbaseConf, TEST_TABLE);
+
+ // Turn off autoflush
+ table.setAutoFlush(false);
+
+ Transaction t = tm.begin();
+ Put put = new Put(row);
+ put.add(family, col, data);
+ table.put(t, put);
+
+ // Data shouldn't be in DB yet
+ Get get = new Get(row);
+ Result result = table.getHTable().get(get);
+ assertEquals("Writes are already in DB", 0, result.size());
+
+ tm.commit(t);
+
+ // After commit, both the cell and shadow cell should be there.
+ // That's why we check for two elements in the test assertion
+ result = table.getHTable().get(get);
+ assertEquals("Writes were not flushed to DB", 2, result.size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestBasicTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestBasicTransaction.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestBasicTransaction.java
new file mode 100644
index 0000000..504710d
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestBasicTransaction.java
@@ -0,0 +1,440 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import static org.junit.Assert.fail;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestBasicTransaction extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestBasicTransaction.class);
+
+
+ @Test(timeOut = 30_000)
+ public void testTimestampsOfTwoRowsInstertedAfterCommitOfSingleTransactionAreEquals(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ byte[] rowName1 = Bytes.toBytes("row1");
+ byte[] rowName2 = Bytes.toBytes("row2");
+ byte[] famName1 = Bytes.toBytes(TEST_FAMILY);
+ byte[] colName1 = Bytes.toBytes("col1");
+ byte[] dataValue1 = Bytes.toBytes("testWrite-1");
+ byte[] dataValue2 = Bytes.toBytes("testWrite-2");
+
+ Transaction tx1 = tm.begin();
+
+ Put row1 = new Put(rowName1);
+ row1.add(famName1, colName1, dataValue1);
+ tt.put(tx1, row1);
+ Put row2 = new Put(rowName2);
+ row2.add(famName1, colName1, dataValue2);
+ tt.put(tx1, row2);
+
+ tm.commit(tx1);
+
+ tt.close();
+
+ // Checks
+ Get getResultRow1 = new Get(rowName1).setMaxVersions(1);
+ Result result1 = tt.getHTable().get(getResultRow1);
+ byte[] val1 = result1.getValue(famName1, colName1);
+ assertTrue(Bytes.equals(dataValue1, result1.getValue(famName1, colName1)),
+ "Unexpected value for row 1 in col 1: " + Bytes.toString(val1));
+ long tsRow1 = result1.rawCells()[0].getTimestamp();
+
+ Get getResultRow2 = new Get(rowName2).setMaxVersions(1);
+ Result result2 = tt.getHTable().get(getResultRow2);
+ byte[] val2 = result2.getValue(famName1, colName1);
+ assertTrue(Bytes.equals(dataValue2, result2.getValue(famName1, colName1)),
+ "Unexpected value for row 2 in col 1: " + Bytes.toString(val2));
+ long tsRow2 = result2.rawCells()[0].getTimestamp();
+
+ assertEquals(tsRow2, tsRow1, "Timestamps of row 1 and row 2 are different");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testTimestampsOfTwoRowsModifiedByTwoSequentialTransactionsAreEqualAndHaveBeenIncreasedMonotonically(ITestContext context)
+ throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ byte[] rowName1 = Bytes.toBytes("row1");
+ byte[] rowName2 = Bytes.toBytes("row2");
+ byte[] famName1 = Bytes.toBytes(TEST_FAMILY);
+ byte[] colName1 = Bytes.toBytes("col1");
+ byte[] dataValue1 = Bytes.toBytes("testWrite-1");
+ byte[] dataValue2 = Bytes.toBytes("testWrite-2");
+
+ byte[] dataValue3 = Bytes.toBytes("testWrite-3");
+ byte[] dataValue4 = Bytes.toBytes("testWrite-4");
+
+ Transaction tx1 = tm.begin();
+
+ Put row1 = new Put(rowName1);
+ row1.add(famName1, colName1, dataValue1);
+ tt.put(tx1, row1);
+ Put row2 = new Put(rowName2);
+ row2.add(famName1, colName1, dataValue2);
+ tt.put(tx1, row2);
+
+ tm.commit(tx1);
+
+ Transaction tx2 = tm.begin();
+
+ row1 = new Put(rowName1);
+ row1.add(famName1, colName1, dataValue3);
+ tt.put(tx2, row1);
+ row2 = new Put(rowName2);
+ row2.add(famName1, colName1, dataValue4);
+ tt.put(tx2, row2);
+
+ tm.commit(tx2);
+
+ tt.close();
+
+ // Checks
+ Get getResultRow1 = new Get(rowName1).setMaxVersions(2);
+ Result result1 = tt.getHTable().get(getResultRow1);
+ byte[] val1 = result1.getValue(famName1, colName1);
+ assertTrue(Bytes.equals(dataValue3, result1.getValue(famName1, colName1)),
+ "Unexpected value for row 1 in col 1: " + Bytes.toString(val1));
+
+ long lastTsRow1 = result1.rawCells()[0].getTimestamp();
+ long previousTsRow1 = result1.rawCells()[1].getTimestamp();
+
+ Get getResultRow2 = new Get(rowName2).setMaxVersions(2);
+ Result result2 = tt.getHTable().get(getResultRow2);
+ byte[] val2 = result2.getValue(famName1, colName1);
+ assertTrue(Bytes.equals(dataValue4, result2.getValue(famName1, colName1)),
+ "Unexpected value for row 2 in col 1: " + Bytes.toString(val2));
+
+ long lastTsRow2 = result2.rawCells()[0].getTimestamp();
+ long previousTsRow2 = result2.rawCells()[1].getTimestamp();
+
+ assertTrue(lastTsRow1 == lastTsRow2, "Timestamps assigned by Tx2 to row 1 and row 2 are different");
+ assertTrue(previousTsRow1 == previousTsRow2, "Timestamps assigned by Tx2 to row 1 and row 2 are different");
+ assertTrue(lastTsRow1 > previousTsRow1, "Timestamp assigned by Tx2 to row 1 hasn't increased monotonically");
+ assertTrue(lastTsRow2 > previousTsRow2, "Timestamp assigned by Tx2 to row 2 hasn't increased monotonically");
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void runTestSimple(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ byte[] row = Bytes.toBytes("test-simple");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+ tm.commit(t1);
+
+ Transaction tread = tm.begin();
+ Transaction t2 = tm.begin();
+ p = new Put(row);
+ p.add(fam, col, data2);
+ tt.put(t2, p);
+ tm.commit(t2);
+
+ Get g = new Get(row).setMaxVersions(1);
+ Result r = tt.getHTable().get(g);
+ assertTrue(Bytes.equals(data2, r.getValue(fam, col)),
+ "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
+
+ r = tt.get(tread, g);
+ assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
+ "Unexpected value for SI read " + tread + ": " + Bytes.toString(r.getValue(fam, col)));
+ }
+
+ @Test(timeOut = 30_000)
+ public void runTestManyVersions(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ byte[] row = Bytes.toBytes("test-simple");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+ tm.commit(t1);
+
+ for (int i = 0; i < 5; ++i) {
+ Transaction t2 = tm.begin();
+ p = new Put(row);
+ p.add(fam, col, data2);
+ tt.put(t2, p);
+ }
+ Transaction tread = tm.begin();
+
+ Get g = new Get(row).setMaxVersions(1);
+ Result r = tt.getHTable().get(g);
+ assertTrue(Bytes.equals(data2, r.getValue(fam, col)),
+ "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
+
+ r = tt.get(tread, g);
+ assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
+ "Unexpected value for SI read " + tread + ": " + Bytes.toString(r.getValue(fam, col)));
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void runTestInterleave(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ byte[] row = Bytes.toBytes("test-interleave");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ tt.put(t1, p);
+ tm.commit(t1);
+
+ Transaction t2 = tm.begin();
+ p = new Put(row);
+ p.add(fam, col, data2);
+ tt.put(t2, p);
+
+ Transaction tread = tm.begin();
+ Get g = new Get(row).setMaxVersions(1);
+ Result r = tt.get(tread, g);
+ assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
+ "Unexpected value for SI read " + tread + ": " + Bytes.toString(r.getValue(fam, col)));
+ tm.commit(t2);
+
+ r = tt.getHTable().get(g);
+ assertTrue(Bytes.equals(data2, r.getValue(fam, col)),
+ "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
+
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, timeOut = 30_000)
+ public void testSameCommitRaisesException(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+
+ Transaction t1 = tm.begin();
+ tm.commit(t1);
+ tm.commit(t1);
+ }
+
+ @Test(timeOut = 30_000)
+ public void testInterleavedScanReturnsTheRightSnapshotResults(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // Basic data-scaffolding for test
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("TEST_COL");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ byte[] startRow = Bytes.toBytes("row-to-scan" + 0);
+ byte[] stopRow = Bytes.toBytes("row-to-scan" + 9);
+ byte[] randomRow = Bytes.toBytes("row-to-scan" + 3);
+
+ // Add some data transactionally to have an initial state for the test
+ Transaction tx1 = tm.begin();
+ for (int i = 0; i < 10; i++) {
+ byte[] row = Bytes.toBytes("row-to-scan" + i);
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ txTable.put(tx1, p);
+ }
+ tm.commit(tx1);
+
+ // Start a second transaction -Tx2- modifying a random row and check that a concurrent transactional context
+ // that scans the table, gets the proper snapshot with the stuff written by Tx1
+ Transaction tx2 = tm.begin();
+ Put p = new Put(randomRow);
+ p.add(fam, col, data2);
+ txTable.put(tx2, p);
+
+ Transaction scanTx = tm.begin(); // This is the concurrent transactional scanner
+ ResultScanner rs = txTable.getScanner(scanTx, new Scan().setStartRow(startRow).setStopRow(stopRow));
+ Result r = rs.next(); // Exercise the next() method
+ int i = 0;
+ while (r != null) {
+ LOG.trace("Scan (" + ++i + ")" + Bytes.toString(r.getRow()) + " => " + Bytes.toString(r.getValue(fam, col)));
+ assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
+ "Unexpected value for SI scan " + scanTx + ": " + Bytes.toString(r.getValue(fam, col)));
+ r = rs.next();
+ }
+
+ // Commit the Tx2 and then check that under a new transactional context, the scanner gets the right snapshot,
+ // which must include the row modified by Tx2
+ tm.commit(tx2);
+
+ int modifiedRows = 0;
+ Transaction newScanTx = tm.begin();
+ ResultScanner newRS = txTable.getScanner(newScanTx, new Scan().setStartRow(startRow).setStopRow(stopRow));
+ Result[] results = newRS.next(10); // Exercise the next(numRows) method
+ for (Result result : results) {
+ if (Bytes.equals(data2, result.getValue(fam, col))) {
+ LOG.trace("Modified :" + Bytes.toString(result.getRow()));
+ modifiedRows++;
+ }
+ }
+ assertEquals(modifiedRows, 1, "Expected 1 row modified, but " + modifiedRows + " are.");
+
+ // Same check as before but checking that the results are correct when retrieved through the Scanner Iterator
+ modifiedRows = 0;
+ ResultScanner iterableRS = txTable.getScanner(newScanTx, new Scan().setStartRow(startRow).setStopRow(stopRow));
+ for (Result res : iterableRS) {
+ if (Bytes.equals(data2, res.getValue(fam, col))) {
+ LOG.trace("Modified :" + Bytes.toString(res.getRow()));
+ modifiedRows++;
+ }
+ }
+
+ assertEquals(modifiedRows, 1, "Expected 1 row modified, but " + modifiedRows + " are.");
+
+ // Finally, check that the Scanner Iterator does not implement the remove method
+ try {
+ iterableRS.iterator().remove();
+ fail();
+ } catch (RuntimeException re) {
+ // Expected
+ }
+
+ }
+
+ @Test(timeOut = 30_000)
+ public void testInterleavedScanReturnsTheRightSnapshotResultsWhenATransactionAborts(ITestContext context)
+ throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ // Basic data-scaffolding for test
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("TEST_COL");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ byte[] startRow = Bytes.toBytes("row-to-scan" + 0);
+ byte[] stopRow = Bytes.toBytes("row-to-scan" + 9);
+ byte[] randomRow = Bytes.toBytes("row-to-scan" + 3);
+
+ // Add some data transactionally to have an initial state for the test
+ Transaction tx1 = tm.begin();
+ for (int i = 0; i < 10; i++) {
+ byte[] row = Bytes.toBytes("row-to-scan" + i);
+
+ Put p = new Put(row);
+ p.add(fam, col, data1);
+ txTable.put(tx1, p);
+ }
+ tm.commit(tx1);
+
+ // Start a second transaction modifying a random row and check that a transactional scanner in Tx2 gets the
+ // right snapshot with the new value in the random row just written by Tx2
+ Transaction tx2 = tm.begin();
+ Put p = new Put(randomRow);
+ p.add(fam, col, data2);
+ txTable.put(tx2, p);
+
+ int modifiedRows = 0;
+ ResultScanner rs = txTable.getScanner(tx2, new Scan().setStartRow(startRow).setStopRow(stopRow));
+ Result r = rs.next();
+ while (r != null) {
+ if (Bytes.equals(data2, r.getValue(fam, col))) {
+ LOG.trace("Modified :" + Bytes.toString(r.getRow()));
+ modifiedRows++;
+ }
+
+ r = rs.next();
+ }
+
+ assertEquals(modifiedRows, 1, "Expected 1 row modified, but " + modifiedRows + " are.");
+
+ // Rollback the second transaction and then check that under a new transactional scanner we get the snapshot
+ // that includes the only the initial rows put by Tx1
+ tm.rollback(tx2);
+
+ Transaction txScan = tm.begin();
+ rs = txTable.getScanner(txScan, new Scan().setStartRow(startRow).setStopRow(stopRow));
+ r = rs.next();
+ while (r != null) {
+ LOG.trace("Scan1 :" + Bytes.toString(r.getRow()) + " => " + Bytes.toString(r.getValue(fam, col)));
+ assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
+ "Unexpected value for SI scan " + txScan + ": " + Bytes.toString(r.getValue(fam, col)));
+ r = rs.next();
+ }
+
+ // Same check as before but checking that the results are correct when retrieved through the Scanner Iterator
+ ResultScanner iterableRS = txTable.getScanner(txScan, new Scan().setStartRow(startRow).setStopRow(stopRow));
+ for (Result result : iterableRS) {
+ assertTrue(Bytes.equals(data1, result.getValue(fam, col)),
+ "Unexpected value for SI scan " + txScan + ": " + Bytes.toString(result.getValue(fam, col)));
+ }
+
+ // Finally, check that the Scanner Iterator does not implement the remove method
+ try {
+ iterableRS.iterator().remove();
+ fail();
+ } catch (RuntimeException re) {
+ // Expected
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestCellUtils.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestCellUtils.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestCellUtils.java
new file mode 100644
index 0000000..7801d0c
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestCellUtils.java
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+import org.apache.omid.HBaseShims;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+
+import static org.apache.omid.transaction.CellUtils.SHADOW_CELL_SUFFIX;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
+import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.AssertJUnit.fail;
+
+@Test(groups = "noHBase")
+public class TestCellUtils {
+
+ private final byte[] row = Bytes.toBytes("test-row");
+ private final byte[] family = Bytes.toBytes("test-family");
+ private final byte[] qualifier = Bytes.toBytes("test-qual");
+ private final byte[] otherQualifier = Bytes.toBytes("other-test-qual");
+
+ @DataProvider(name = "shadow-cell-suffixes")
+ public Object[][] createShadowCellSuffixes() {
+ return new Object[][]{
+ {SHADOW_CELL_SUFFIX},
+ };
+ }
+
+ @Test(dataProvider = "shadow-cell-suffixes")
+ public void testShadowCellQualifiers(byte[] shadowCellSuffixToTest) throws IOException {
+
+ final byte[] validShadowCellQualifier =
+ com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
+ final byte[] sandwichValidShadowCellQualifier =
+ com.google.common.primitives.Bytes.concat(shadowCellSuffixToTest, validShadowCellQualifier);
+ final byte[] doubleEndedValidShadowCellQualifier =
+ com.google.common.primitives.Bytes.concat(validShadowCellQualifier, shadowCellSuffixToTest);
+ final byte[] interleavedValidShadowCellQualifier =
+ com.google.common.primitives.Bytes.concat(validShadowCellQualifier,
+ com.google.common.primitives.Bytes
+ .concat(validShadowCellQualifier, validShadowCellQualifier));
+ final byte[] value = Bytes.toBytes("test-value");
+
+ // Test the qualifier passed is a shadow cell
+ // qualifier because it contains only one suffix
+ // and is placed at the end of the qualifier:
+ // qual_nameSUFFIX
+ KeyValue kv = new KeyValue(row, family, validShadowCellQualifier, value);
+ assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
+
+ // We also accept this pattern in the qualifier:
+ // SUFFIXqual_nameSUFFIX
+ kv = new KeyValue(row, family, sandwichValidShadowCellQualifier, value);
+ assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
+
+ // We also accept this pattern in the qualifier:
+ // qual_nameSUFFIXSUFFIX
+ kv = new KeyValue(row, family, doubleEndedValidShadowCellQualifier, value);
+ assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
+
+ // We also accept this pattern in the qualifier:
+ // qual_nameSUFFIXqual_nameSUFFIXqual_nameSUFFIX
+ kv = new KeyValue(row, family, interleavedValidShadowCellQualifier, value);
+ assertTrue("Should include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
+
+ // Test the qualifier passed is not a shadow cell
+ // qualifier if there's nothing else apart from the suffix
+ kv = new KeyValue(row, family, shadowCellSuffixToTest, value);
+ assertFalse("Should not include a valid shadowCell identifier", CellUtils.isShadowCell(kv));
+
+ }
+
+ @Test
+ public void testCorrectMapingOfCellsToShadowCells() throws IOException {
+ // Create the required data
+ final byte[] validShadowCellQualifier =
+ com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
+
+ final byte[] qualifier2 = Bytes.toBytes("test-qual2");
+ final byte[] validShadowCellQualifier2 =
+ com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX);
+
+ final byte[] qualifier3 = Bytes.toBytes("test-qual3");
+
+ Cell cell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
+ Cell dupCell1 = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
+ Cell dupCell1WithAnotherValue = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
+ Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
+ Cell shadowCell1 = new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));
+
+ Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
+ Cell shadowCell2 = new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));
+
+ Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));
+
+ // Check a list of cells with duplicate values
+ List<Cell> badListWithDups = new ArrayList<>();
+ badListWithDups.add(cell1);
+ badListWithDups.add(dupCell1WithAnotherValue);
+
+ // Check dup shadow cell with same MVCC is ignored
+ SortedMap<Cell, Optional<Cell>> cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
+ assertEquals("There should be only 1 key-value maps", 1, cellsToShadowCells.size());
+ assertTrue(cellsToShadowCells.containsKey(cell1));
+ KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
+ KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
+ assertTrue(firstKey.equals(lastKey));
+ assertTrue("Should be equal", 0 == Bytes.compareTo(
+ firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
+ cell1.getValueArray(), cell1.getValueOffset(), cell1.getValueLength()));
+
+ // Modify dup shadow cell to have a greater MVCC and check that is replaced
+ HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
+ cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
+ assertEquals("There should be only 1 key-value maps", 1, cellsToShadowCells.size());
+ assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
+ firstKey = (KeyValue) cellsToShadowCells.firstKey();
+ lastKey = (KeyValue) cellsToShadowCells.lastKey();
+ assertTrue(firstKey.equals(lastKey));
+ assertTrue("Should be equal", 0 == Bytes.compareTo(
+ firstKey.getValueArray(), firstKey.getValueOffset(), firstKey.getValueLength(),
+ dupCell1WithAnotherValue.getValueArray(), dupCell1WithAnotherValue.getValueOffset(),
+ dupCell1WithAnotherValue.getValueLength()));
+ // Check a list of cells with duplicate values
+ List<Cell> cellListWithDups = new ArrayList<>();
+ cellListWithDups.add(cell1);
+ cellListWithDups.add(shadowCell1);
+ cellListWithDups.add(dupCell1); // Dup cell
+ cellListWithDups.add(delCell1); // Another Dup cell but with different type
+ cellListWithDups.add(cell2);
+ cellListWithDups.add(cell3);
+ cellListWithDups.add(shadowCell2);
+
+ cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
+ assertEquals("There should be only 3 key-value maps", 3, cellsToShadowCells.size());
+ assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
+ assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
+ assertFalse(cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
+ // The current algo avoids to put the delete cell
+ // as key after the put cell with same value was added
+ assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
+ assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
+
+ }
+
+ @Test
+ public void testShadowCellSuffixConcatenationToQualifier() {
+
+ Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
+ byte[] suffixedQualifier = CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ byte[] expectedQualifier = com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);
+ assertEquals(expectedQualifier, suffixedQualifier);
+
+ }
+
+ @Test(dataProvider = "shadow-cell-suffixes")
+ public void testShadowCellSuffixRemovalFromQualifier(byte[] shadowCellSuffixToTest) throws IOException {
+
+ // Test removal from a correclty suffixed qualifier
+ byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
+ Cell cell = new KeyValue(row, family, suffixedQualifier, 1, Bytes.toBytes("value"));
+ byte[] resultedQualifier = CellUtils.removeShadowCellSuffix(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ byte[] expectedQualifier = qualifier;
+ assertEquals(expectedQualifier, resultedQualifier);
+
+ // Test removal from a badly suffixed qualifier
+ byte[] badlySuffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, Bytes.toBytes("BAD"));
+ Cell badCell = new KeyValue(row, family, badlySuffixedQualifier, 1, Bytes.toBytes("value"));
+ try {
+ CellUtils.removeShadowCellSuffix(badCell.getQualifierArray(),
+ badCell.getQualifierOffset(),
+ badCell.getQualifierLength());
+ fail();
+ } catch (IllegalArgumentException e) {
+ // Expected
+ }
+ }
+
+ @Test
+ public void testMatchingQualifiers() {
+ Cell cell = new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value"));
+ assertTrue(CellUtils.matchingQualifier(cell, qualifier, 0, qualifier.length));
+ assertFalse(CellUtils.matchingQualifier(cell, otherQualifier, 0, otherQualifier.length));
+ }
+
+ @Test(dataProvider = "shadow-cell-suffixes")
+ public void testQualifierLengthFromShadowCellQualifier(byte[] shadowCellSuffixToTest) {
+ // Test suffixed qualifier
+ byte[] suffixedQualifier = com.google.common.primitives.Bytes.concat(qualifier, shadowCellSuffixToTest);
+ int originalQualifierLength =
+ CellUtils.qualifierLengthFromShadowCellQualifier(suffixedQualifier, 0, suffixedQualifier.length);
+ assertEquals(qualifier.length, originalQualifierLength);
+
+ // Test passing qualifier without shadow cell suffix
+ originalQualifierLength =
+ CellUtils.qualifierLengthFromShadowCellQualifier(qualifier, 0, qualifier.length);
+ assertEquals(qualifier.length, originalQualifierLength);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestColumnIterator.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestColumnIterator.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestColumnIterator.java
new file mode 100644
index 0000000..02287f4
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestColumnIterator.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.mortbay.log.Log;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static org.testng.AssertJUnit.assertEquals;
+
+@Test(groups = "noHBase")
+public class TestColumnIterator {
+
+ final byte[] row = Bytes.toBytes("row");
+ private final byte[] family1 = Bytes.toBytes("f1");
+ private final byte[] family2 = Bytes.toBytes("f2");
+ private final byte[] qualifier1 = Bytes.toBytes("c1");
+ private final byte[] qualifier2 = Bytes.toBytes("c2");
+ final byte[] data = Bytes.toBytes("data");
+
+ private final List<Cell> cells = new ArrayList<Cell>(
+ Arrays.asList(
+ // Group 1 (3 elems but grouping should filter shadow cell, so check for 2)
+ new KeyValue(row, family1, qualifier1, 0, data),
+ new KeyValue(row, family1, qualifier1, 1, data),
+ new KeyValue(row, family1, CellUtils.addShadowCellSuffix(qualifier1), 0, data),
+ // Group 2 (2 elems but grouping should filter shadow cell, so check for 1)
+ new KeyValue(row, family1, qualifier2, 0, data),
+ new KeyValue(row, family1, CellUtils.addShadowCellSuffix(qualifier2), 0, data),
+ // Group 3 (2 elems but grouping should filter shadow cell, so check for 1)
+ new KeyValue(row, family2, qualifier1, 0, data),
+ new KeyValue(row, family2, CellUtils.addShadowCellSuffix(qualifier1), 0, data)
+ )
+ );
+
+ @Test
+ public void testGroupingCellsByColumnFilteringShadowCells() {
+
+ ImmutableList<Collection<Cell>> groupedColumnsWithoutShadowCells =
+ TTable.groupCellsByColumnFilteringShadowCells(cells);
+ Log.info("Column Groups " + groupedColumnsWithoutShadowCells);
+ assertEquals("Should be 3 column groups", 3, groupedColumnsWithoutShadowCells.size());
+ int group1Counter = 0;
+ int group2Counter = 0;
+ int group3Counter = 0;
+ for (Collection<Cell> columns : groupedColumnsWithoutShadowCells) {
+ for (Cell cell : columns) {
+ byte[] cellFamily = CellUtil.cloneFamily(cell);
+ byte[] cellQualifier = CellUtil.cloneQualifier(cell);
+ // Group 1
+ if (Bytes.equals(cellFamily, family1) &&
+ Bytes.equals(cellQualifier, qualifier1)) {
+ group1Counter++;
+ }
+ // Group 2
+ if (Bytes.equals(cellFamily, family1) &&
+ Bytes.equals(cellQualifier, qualifier2)) {
+ group2Counter++;
+ }
+ // Group 3
+ if (Bytes.equals(cellFamily, family2) &&
+ Bytes.equals(cellQualifier, qualifier1)) {
+ group3Counter++;
+ }
+ }
+ }
+
+ assertEquals("Group 1 should have 2 elems", 2, group1Counter);
+ assertEquals("Group 2 should have 1 elems", 1, group2Counter);
+ assertEquals("Group 3 should have 1 elems", 1, group3Counter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/org/apache/omid/transaction/TestDeletion.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestDeletion.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestDeletion.java
new file mode 100644
index 0000000..201cbfb
--- /dev/null
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestDeletion.java
@@ -0,0 +1,299 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.AssertJUnit;
+import org.testng.ITestContext;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.testng.Assert.assertTrue;
+
+@Test(groups = "sharedHBase")
+public class TestDeletion extends OmidTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestDeletion.class);
+
+ private byte[] famA = Bytes.toBytes(TEST_FAMILY);
+ private byte[] famB = Bytes.toBytes(TEST_FAMILY2);
+ private byte[] colA = Bytes.toBytes("testdataA");
+ private byte[] colB = Bytes.toBytes("testdataB");
+ private byte[] data1 = Bytes.toBytes("testWrite-1");
+ private byte[] modrow = Bytes.toBytes("test-del" + 3);
+
+ private static class FamCol {
+
+ final byte[] fam;
+ final byte[] col;
+
+ FamCol(byte[] fam, byte[] col) {
+ this.fam = fam;
+ this.col = col;
+ }
+
+ }
+
+ @Test
+ public void runTestDeleteFamily(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ int rowsWritten = 10;
+ FamCol famColA = new FamCol(famA, colA);
+ FamCol famColB = new FamCol(famB, colB);
+ writeRows(tt, t1, rowsWritten, famColA, famColB);
+ tm.commit(t1);
+
+ Transaction t2 = tm.begin();
+ Delete d = new Delete(modrow);
+ d.deleteFamily(famA);
+ tt.delete(t2, d);
+
+ Transaction tscan = tm.begin();
+ ResultScanner rs = tt.getScanner(tscan, new Scan());
+
+ Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
+ AssertJUnit.assertEquals("ColA count should be equal to rowsWritten", rowsWritten, (int) count.get(famColA));
+ AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
+ tm.commit(t2);
+
+ tscan = tm.begin();
+ rs = tt.getScanner(tscan, new Scan());
+
+ count = countColsInRows(rs, famColA, famColB);
+ AssertJUnit
+ .assertEquals("ColA count should be equal to rowsWritten - 1", (rowsWritten - 1), (int) count.get(famColA));
+ AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
+ }
+
+ @Test
+ public void runTestDeleteColumn(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ int rowsWritten = 10;
+
+ FamCol famColA = new FamCol(famA, colA);
+ FamCol famColB = new FamCol(famA, colB);
+ writeRows(tt, t1, rowsWritten, famColA, famColB);
+ tm.commit(t1);
+
+ Transaction t2 = tm.begin();
+ Delete d = new Delete(modrow);
+ d.deleteColumn(famA, colA);
+ tt.delete(t2, d);
+
+ Transaction tscan = tm.begin();
+ ResultScanner rs = tt.getScanner(tscan, new Scan());
+
+ Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
+ AssertJUnit.assertEquals("ColA count should be equal to rowsWritten", rowsWritten, (int) count.get(famColA));
+ AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
+ tm.commit(t2);
+
+ tscan = tm.begin();
+ rs = tt.getScanner(tscan, new Scan());
+
+ count = countColsInRows(rs, famColA, famColB);
+ AssertJUnit
+ .assertEquals("ColA count should be equal to rowsWritten - 1", (rowsWritten - 1), (int) count.get(famColA));
+ AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
+ }
+
+ /**
+ * This test is very similar to #runTestDeleteColumn() but exercises Delete#deleteColumns()
+ */
+ @Test
+ public void runTestDeleteColumns(ITestContext context) throws Exception {
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ int rowsWritten = 10;
+
+ FamCol famColA = new FamCol(famA, colA);
+ FamCol famColB = new FamCol(famA, colB);
+ writeRows(tt, t1, rowsWritten, famColA, famColB);
+ tm.commit(t1);
+
+ Transaction t2 = tm.begin();
+ Delete d = new Delete(modrow);
+ d.deleteColumns(famA, colA);
+ tt.delete(t2, d);
+
+ Transaction tscan = tm.begin();
+ ResultScanner rs = tt.getScanner(tscan, new Scan());
+
+ Map<FamCol, Integer> count = countColsInRows(rs, famColA, famColB);
+ AssertJUnit.assertEquals("ColA count should be equal to rowsWritten", rowsWritten, (int) count.get(famColA));
+ AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
+ tm.commit(t2);
+
+ tscan = tm.begin();
+ rs = tt.getScanner(tscan, new Scan());
+
+ count = countColsInRows(rs, famColA, famColB);
+
+ AssertJUnit
+ .assertEquals("ColA count should be equal to rowsWritten - 1", (rowsWritten - 1), (int) count.get(famColA));
+ AssertJUnit.assertEquals("ColB count should be equal to rowsWritten", rowsWritten, (int) count.get(famColB));
+ }
+
+ @Test
+ public void runTestDeleteRow(ITestContext context) throws Exception {
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ int rowsWritten = 10;
+
+ FamCol famColA = new FamCol(famA, colA);
+ writeRows(tt, t1, rowsWritten, famColA);
+
+ tm.commit(t1);
+
+ Transaction t2 = tm.begin();
+ Delete d = new Delete(modrow);
+ tt.delete(t2, d);
+
+ Transaction tscan = tm.begin();
+ ResultScanner rs = tt.getScanner(tscan, new Scan());
+
+ int rowsRead = countRows(rs);
+ AssertJUnit.assertTrue("Expected " + rowsWritten + " rows but " + rowsRead + " found",
+ rowsRead == rowsWritten);
+
+ tm.commit(t2);
+
+ tscan = tm.begin();
+ rs = tt.getScanner(tscan, new Scan());
+
+ rowsRead = countRows(rs);
+ AssertJUnit.assertTrue("Expected " + (rowsWritten - 1) + " rows but " + rowsRead + " found",
+ rowsRead == (rowsWritten - 1));
+
+ }
+
+ @Test
+ public void testDeletionOfNonExistingColumnFamilyDoesNotWriteToHBase(ITestContext context) throws Exception {
+
+ // --------------------------------------------------------------------
+ // Setup initial environment for the test
+ // --------------------------------------------------------------------
+ TransactionManager tm = newTransactionManager(context);
+ TTable txTable = new TTable(hbaseConf, TEST_TABLE);
+
+ Transaction tx1 = tm.begin();
+ LOG.info("{} writing initial data created ", tx1);
+ Put p = new Put(Bytes.toBytes("row1"));
+ p.add(famA, colA, data1);
+ txTable.put(tx1, p);
+ tm.commit(tx1);
+
+ // --------------------------------------------------------------------
+ // Try to delete a non existing CF
+ // --------------------------------------------------------------------
+ Transaction deleteTx = tm.begin();
+ LOG.info("{} trying to delete a non-existing family created ", deleteTx);
+ Delete del = new Delete(Bytes.toBytes("row1"));
+ del.deleteFamily(famB);
+ // This delete should not put data on HBase
+ txTable.delete(deleteTx, del);
+
+ // --------------------------------------------------------------------
+ // Check data has not been written to HBase
+ // --------------------------------------------------------------------
+ HTable table = new HTable(hbaseConf, TEST_TABLE);
+ Get get = new Get(Bytes.toBytes("row1"));
+ get.setTimeStamp(deleteTx.getTransactionId());
+ Result result = table.get(get);
+ assertTrue(result.isEmpty());
+
+ }
+
+ private int countRows(ResultScanner rs) throws IOException {
+ int count;
+ Result r = rs.next();
+ count = 0;
+ while (r != null) {
+ count++;
+ LOG.trace("row: " + Bytes.toString(r.getRow()) + " count: " + count);
+ r = rs.next();
+ }
+ return count;
+ }
+
+ private void writeRows(TTable tt, Transaction t1, int rowcount, FamCol... famCols) throws IOException {
+ for (int i = 0; i < rowcount; i++) {
+ byte[] row = Bytes.toBytes("test-del" + i);
+
+ Put p = new Put(row);
+ for (FamCol col : famCols) {
+ p.add(col.fam, col.col, data1);
+ }
+ tt.put(t1, p);
+ }
+ }
+
+ private Map<FamCol, Integer> countColsInRows(ResultScanner rs, FamCol... famCols) throws IOException {
+ Map<FamCol, Integer> colCount = new HashMap<>();
+ Result r = rs.next();
+ while (r != null) {
+ for (FamCol col : famCols) {
+ if (r.containsColumn(col.fam, col.col)) {
+ Integer c = colCount.get(col);
+
+ if (c == null) {
+ colCount.put(col, 1);
+ } else {
+ colCount.put(col, c + 1);
+ }
+ }
+ }
+ r = rs.next();
+ }
+ return colCount;
+ }
+
+}
[50/52] [abbrv] incubator-omid git commit: Add code quality reports:
checkstyle and PMD
Posted by ik...@apache.org.
Add code quality reports: checkstyle and PMD
Change-Id: I7500d5fdc5f917a0362252d8ea5ac26049bcd94c
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/4f3e4c2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/4f3e4c2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/4f3e4c2f
Branch: refs/heads/master
Commit: 4f3e4c2f936f5a725de4c614df061351491553ea
Parents: 410653e
Author: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Authored: Wed Apr 20 10:50:18 2016 -0700
Committer: Francisco Perez-Sorrosal <fp...@yahoo-inc.com>
Committed: Wed Apr 20 10:50:18 2016 -0700
----------------------------------------------------------------------
doc/site/markdown/coding-guide-and-style.md | 10 +-
misc/omid_checks.xml | 211 +++++++++++++++++++++++
pom.xml | 37 ++++
3 files changed, 255 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/4f3e4c2f/doc/site/markdown/coding-guide-and-style.md
----------------------------------------------------------------------
diff --git a/doc/site/markdown/coding-guide-and-style.md b/doc/site/markdown/coding-guide-and-style.md
index 504094f..e751878 100644
--- a/doc/site/markdown/coding-guide-and-style.md
+++ b/doc/site/markdown/coding-guide-and-style.md
@@ -16,17 +16,17 @@ The basic principle is to always write code that is testable, easy to understand
* Use proven libraries for (e.g. Apache commons, guava, etc.)
* Refactor when necessary
* When adding a new feature and the context is not appropriate, refactor first in a separate commit/s
+* Use [TestNG](http://testng.org/) for testing instead of JUnit
# Coding Style
-Omid coding style shoud follow [Oracle's Code Conventions for Java](http://www.oracle.com/technetwork/java/javase/documentation/codeconvtoc-136057.html), with the following modifications:
+Omid coding style should follow general rules found in [Google's Code Style for Java](https://google.github.io/styleguide/javaguide.html), with the following modifications:
* Lines can be up to 120 characters long
-* Indentation should be:
+* Block indentation should be:
* 4 spaces
* Tabs not allowed
* Always use curly braces for code blocks, even for single-line 'ifs' and 'elses'
* Do not include @author tags in any javadoc
-* Use TestNG for testing
* Import ordering and spacing:
* Try to organize imports alphabetically in blocks with this format:
* A first block of imports from external libraries
@@ -54,3 +54,7 @@ Omid coding style shoud follow [Oracle's Code Conventions for Java](http://www.o
import static com.yahoo.omid.zk.ZKUtils.provideZookeeperClient;
...
```
+
+The complete rules (specified with [Checkstyle](https://github.com/checkstyle/checkstyle)) can be found in the
+`misc/omid_checks.xml` file. A summary of the current status of the style in the Omid project can be found in
+the [[checkstyle-aggregate]] page.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/4f3e4c2f/misc/omid_checks.xml
----------------------------------------------------------------------
diff --git a/misc/omid_checks.xml b/misc/omid_checks.xml
new file mode 100644
index 0000000..e69d26f
--- /dev/null
+++ b/misc/omid_checks.xml
@@ -0,0 +1,211 @@
+<?xml version="1.0"?>
+<!DOCTYPE module PUBLIC
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+
+<!--
+ Checkstyle configuration for Omid based on the Google coding conventions
+ from Google Java Style: https://google.github.io/styleguide/javaguide.html.
+
+ Omid's style main modifications in Modules below:
+ - LineLength & LeftCurly.maxLineLength: 120 chars instead of 100
+ - Indentation params: 4 spaces instead of 2
+ - AbbreviationAsWordInName.allowedAbbreviationLength: 4 instead of 1
+ -->
+
+<module name = "Checker">
+
+ <property name="charset" value="UTF-8"/>
+
+ <property name="severity" value="warning"/>
+
+ <property name="fileExtensions" value="java, properties, xml"/>
+ <!-- Checks for whitespace -->
+ <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+ <module name="FileTabCharacter">
+ <property name="eachLine" value="true"/>
+ </module>
+
+ <module name="TreeWalker">
+ <module name="OuterTypeFilename"/>
+ <module name="IllegalTokenText">
+ <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
+ <property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
+ <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
+ </module>
+ <module name="AvoidEscapedUnicodeCharacters">
+ <property name="allowEscapesForControlCharacters" value="true"/>
+ <property name="allowByTailComment" value="true"/>
+ <property name="allowNonPrintableEscapes" value="true"/>
+ </module>
+ <module name="LineLength">
+ <property name="max" value="120"/>
+ <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
+ </module>
+ <module name="AvoidStarImport"/>
+ <module name="OneTopLevelClass"/>
+ <module name="NoLineWrap"/>
+ <module name="EmptyBlock">
+ <property name="option" value="TEXT"/>
+ <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
+ </module>
+ <module name="NeedBraces"/>
+ <module name="LeftCurly">
+ <property name="maxLineLength" value="120"/>
+ </module>
+ <module name="RightCurly"/>
+ <module name="RightCurly">
+ <property name="option" value="alone"/>
+ <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, LITERAL_DO, STATIC_INIT, INSTANCE_INIT"/>
+ </module>
+ <module name="WhitespaceAround">
+ <property name="allowEmptyConstructors" value="true"/>
+ <property name="allowEmptyMethods" value="true"/>
+ <property name="allowEmptyTypes" value="true"/>
+ <property name="allowEmptyLoops" value="true"/>
+ <message key="ws.notFollowed"
+ value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
+ <message key="ws.notPreceded"
+ value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
+ </module>
+ <module name="OneStatementPerLine"/>
+ <module name="MultipleVariableDeclarations"/>
+ <module name="ArrayTypeStyle"/>
+ <module name="MissingSwitchDefault"/>
+ <module name="FallThrough"/>
+ <module name="UpperEll"/>
+ <module name="ModifierOrder"/>
+ <module name="EmptyLineSeparator">
+ <property name="allowNoEmptyLineBetweenFields" value="true"/>
+ </module>
+ <module name="SeparatorWrap">
+ <property name="tokens" value="DOT"/>
+ <property name="option" value="nl"/>
+ </module>
+ <module name="SeparatorWrap">
+ <property name="tokens" value="COMMA"/>
+ <property name="option" value="EOL"/>
+ </module>
+ <module name="PackageName">
+ <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
+ <message key="name.invalidPattern"
+ value="Package name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="TypeName">
+ <message key="name.invalidPattern"
+ value="Type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="MemberName">
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <message key="name.invalidPattern"
+ value="Member name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="ParameterName">
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <message key="name.invalidPattern"
+ value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <!-- The next module is commented cause it shows errors in checkstyle -->
+ <!-- module name="CatchParameterName">
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <message key="name.invalidPattern"
+ value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
+ </module-->
+ <module name="LocalVariableName">
+ <property name="tokens" value="VARIABLE_DEF"/>
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+ <property name="allowOneCharVarInForLoop" value="true"/>
+ <message key="name.invalidPattern"
+ value="Local variable name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="ClassTypeParameterName">
+ <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+ <message key="name.invalidPattern"
+ value="Class type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="MethodTypeParameterName">
+ <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+ <message key="name.invalidPattern"
+ value="Method type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="InterfaceTypeParameterName">
+ <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+ <message key="name.invalidPattern"
+ value="Interface type name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="NoFinalizer"/>
+ <module name="GenericWhitespace">
+ <message key="ws.followed"
+ value="GenericWhitespace ''{0}'' is followed by whitespace."/>
+ <message key="ws.preceded"
+ value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
+ <message key="ws.illegalFollow"
+ value="GenericWhitespace ''{0}'' should followed by whitespace."/>
+ <message key="ws.notPreceded"
+ value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
+ </module>
+ <module name="Indentation">
+ <property name="basicOffset" value="4"/>
+ <property name="braceAdjustment" value="0"/>
+ <property name="caseIndent" value="4"/>
+ <property name="throwsIndent" value="4"/>
+ <property name="lineWrappingIndentation" value="4"/>
+ <property name="arrayInitIndent" value="4"/>
+ </module>
+ <module name="AbbreviationAsWordInName">
+ <property name="ignoreFinal" value="false"/>
+ <property name="allowedAbbreviationLength" value="4"/>
+ </module>
+ <module name="OverloadMethodsDeclarationOrder"/>
+ <module name="VariableDeclarationUsageDistance"/>
+ <module name="CustomImportOrder">
+ <property name="specialImportsRegExp" value="com.google"/>
+ <property name="sortImportsInGroupAlphabetically" value="true"/>
+ <property name="customImportOrderRules" value="STATIC###SPECIAL_IMPORTS###THIRD_PARTY_PACKAGE###STANDARD_JAVA_PACKAGE"/>
+ </module>
+ <module name="MethodParamPad"/>
+ <module name="OperatorWrap">
+ <property name="option" value="NL"/>
+ <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR "/>
+ </module>
+ <module name="AnnotationLocation">
+ <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
+ </module>
+ <module name="AnnotationLocation">
+ <property name="tokens" value="VARIABLE_DEF"/>
+ <property name="allowSamelineMultipleAnnotations" value="true"/>
+ </module>
+ <module name="NonEmptyAtclauseDescription"/>
+ <module name="JavadocTagContinuationIndentation"/>
+ <module name="SummaryJavadoc">
+ <property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
+ </module>
+ <module name="JavadocParagraph"/>
+ <module name="AtclauseOrder">
+ <property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
+ <property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
+ </module>
+ <module name="JavadocMethod">
+ <property name="scope" value="public"/>
+ <property name="allowMissingParamTags" value="true"/>
+ <property name="allowMissingThrowsTags" value="true"/>
+ <property name="allowMissingReturnTag" value="true"/>
+ <property name="minLineCount" value="2"/>
+ <property name="allowedAnnotations" value="Override, Test"/>
+ <property name="allowThrowsTagsForSubclasses" value="true"/>
+ </module>
+ <module name="MethodName">
+ <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
+ <message key="name.invalidPattern"
+ value="Method name ''{0}'' must match pattern ''{1}''."/>
+ </module>
+ <module name="SingleLineJavadoc">
+ <property name="ignoreInlineTags" value="false"/>
+ </module>
+ <module name="EmptyCatchBlock">
+ <property name="exceptionVariableName" value="expected"/>
+ </module>
+ <module name="CommentsIndentation"/>
+ </module>
+
+</module>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/4f3e4c2f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index d74eb0c..0150152 100644
--- a/pom.xml
+++ b/pom.xml
@@ -171,6 +171,9 @@
<maven-cobertura-plugin.version>2.7</maven-cobertura-plugin.version>
<maven-license-plugin.version>2.11</maven-license-plugin.version>
<maven-site-plugin.version>3.5</maven-site-plugin.version>
+ <maven-pmd-plugin.version>3.4</maven-pmd-plugin.version>
+ <maven-checkstyle-plugin.version>2.17</maven-checkstyle-plugin.version>
+ <maven-jxr-plugin.version>2.3</maven-jxr-plugin.version>
<!-- Licensing properties (for license-maven-plugins) -->
<license.header>misc/header.txt</license.header>
@@ -567,4 +570,38 @@
</dependencyManagement>
+ <!-- Generated when executing 'mvn site' -->
+ <reporting>
+
+ <plugins>
+
+ <!-- Required for generating cross references with sources on reports: e.g. line numbers... -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jxr-plugin</artifactId>
+ <version>${maven-jxr-plugin.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <configuration>
+ <configLocation>misc/omid_checks.xml</configLocation>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-pmd-plugin</artifactId>
+ <version>${maven-pmd-plugin.version}</version>
+ <configuration>
+ <aggregate>true</aggregate>
+ </configuration>
+ </plugin>
+
+ </plugins>
+
+ </reporting>
+
</project>
[24/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransactionManager.java b/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransactionManager.java
new file mode 100644
index 0000000..d7215e6
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/AbstractTransactionManager.java
@@ -0,0 +1,502 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.util.concurrent.Futures;
+import org.apache.omid.committable.CommitTable;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.metrics.Counter;
+import org.apache.omid.metrics.MetricsRegistry;
+import org.apache.omid.metrics.Timer;
+import org.apache.omid.transaction.Transaction.Status;
+import org.apache.omid.tso.client.AbortException;
+import org.apache.omid.tso.client.CellId;
+import org.apache.omid.tso.client.ConnectionException;
+import org.apache.omid.tso.client.NewTSOException;
+import org.apache.omid.tso.client.ServiceUnavailableException;
+import org.apache.omid.tso.client.TSOClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.CACHE;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.COMMIT_TABLE;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.NOT_PRESENT;
+import static org.apache.omid.committable.CommitTable.CommitTimestamp.Location.SHADOW_CELL;
+import static org.apache.omid.metrics.MetricsUtils.name;
+
+/**
+ * Omid's base abstract implementation of the {@link TransactionManager} interface.
+ *
+ * Provides extra methods to allow transaction manager developers to perform
+ * different actions before/after the methods exposed by the {@link TransactionManager} interface.
+ *
+ * So, this abstract class must be extended by particular implementations of
+ * transaction managers related to different storage systems (HBase...)
+ */
+public abstract class AbstractTransactionManager implements TransactionManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractTransactionManager.class);
+
+ public interface TransactionFactory<T extends CellId> {
+
+ AbstractTransaction<T> createTransaction(long transactionId, long epoch, AbstractTransactionManager tm);
+
+ }
+
+ private final PostCommitActions postCommitter;
+ protected final TSOClient tsoClient;
+ protected final CommitTable.Client commitTableClient;
+ private final TransactionFactory<? extends CellId> transactionFactory;
+
+ // Metrics
+ private final Timer startTimestampTimer;
+ private final Timer commitTimer;
+ private final Counter committedTxsCounter;
+ private final Counter rolledbackTxsCounter;
+ private final Counter errorTxsCounter;
+ private final Counter invalidatedTxsCounter;
+
+ /**
+ * Base constructor
+ *
+ * @param metrics
+ * instrumentation metrics
+ * @param postCommitter
+ * post commit action executor
+ * @param tsoClient
+ * a client for accessing functionality of the status oracle
+ * @param commitTableClient
+ * a client for accessing functionality of the commit table
+ * @param transactionFactory
+ * a transaction factory to create the specific transaction
+ * objects required by the transaction manager being implemented.
+ */
+ public AbstractTransactionManager(MetricsRegistry metrics,
+ PostCommitActions postCommitter,
+ TSOClient tsoClient,
+ CommitTable.Client commitTableClient,
+ TransactionFactory<? extends CellId> transactionFactory) {
+
+ this.tsoClient = tsoClient;
+ this.postCommitter = postCommitter;
+ this.commitTableClient = commitTableClient;
+ this.transactionFactory = transactionFactory;
+
+ // Metrics configuration
+ this.startTimestampTimer = metrics.timer(name("omid", "tm", "hbase", "startTimestamp", "latency"));
+ this.commitTimer = metrics.timer(name("omid", "tm", "hbase", "commit", "latency"));
+ this.committedTxsCounter = metrics.counter(name("omid", "tm", "hbase", "committedTxs"));
+ this.rolledbackTxsCounter = metrics.counter(name("omid", "tm", "hbase", "rolledbackTxs"));
+ this.errorTxsCounter = metrics.counter(name("omid", "tm", "hbase", "erroredTxs"));
+ this.invalidatedTxsCounter = metrics.counter(name("omid", "tm", "hbase", "invalidatedTxs"));
+
+ }
+
+ /**
+ * Allows transaction manager developers to perform actions before creating a transaction.
+ * @throws TransactionManagerException
+ */
+ public void preBegin() throws TransactionManagerException {}
+
+ /**
+ * @see org.apache.omid.transaction.TransactionManager#begin()
+ */
+ @Override
+ public final Transaction begin() throws TransactionException {
+
+ try {
+ preBegin();
+
+ long startTimestamp, epoch;
+
+ // The loop is required for HA scenarios where we get the timestamp
+ // but when getting the epoch, the client is connected to a new TSOServer
+ // When this happen, the epoch will be larger than the startTimestamp,
+ // so we need to start the transaction again. We use the fact that epoch
+ // is always smaller or equal to a timestamp, and therefore, we first need
+ // to get the timestamp and then the epoch.
+ startTimestampTimer.start();
+ try {
+ do {
+ startTimestamp = tsoClient.getNewStartTimestamp().get();
+ epoch = tsoClient.getEpoch();
+ } while (epoch > startTimestamp);
+ } finally {
+ startTimestampTimer.stop();
+ }
+
+ AbstractTransaction<? extends CellId> tx = transactionFactory.createTransaction(startTimestamp, epoch, this);
+
+ postBegin(tx);
+
+ return tx;
+ } catch (TransactionManagerException e) {
+ throw new TransactionException("An error has occured during PreBegin/PostBegin", e);
+ } catch (ExecutionException e) {
+ throw new TransactionException("Could not get new timestamp", e);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new TransactionException("Interrupted getting timestamp", ie);
+ }
+ }
+
+ /**
+ * Allows transaction manager developers to perform actions after having started a transaction.
+ * @param transaction
+ * the transaction that was just created.
+ * @throws TransactionManagerException
+ */
+ public void postBegin(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
+
+ /**
+ * Allows transaction manager developers to perform actions before committing a transaction.
+ * @param transaction
+ * the transaction that is going to be committed.
+ * @throws TransactionManagerException
+ */
+ public void preCommit(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
+
+ /**
+ * @see org.apache.omid.transaction.TransactionManager#commit(Transaction)
+ */
+ @Override
+ public final void commit(Transaction transaction) throws RollbackException, TransactionException {
+
+ AbstractTransaction<? extends CellId> tx = enforceAbstractTransactionAsParam(transaction);
+ enforceTransactionIsInRunningState(tx);
+
+ if (tx.isRollbackOnly()) { // Manage explicit user rollback
+ rollback(tx);
+ throw new RollbackException(tx + ": Tx was set to rollback explicitly");
+ }
+
+ try {
+
+ preCommit(tx);
+
+ commitTimer.start();
+ try {
+ if (tx.getWriteSet().isEmpty()) {
+ markReadOnlyTransaction(tx); // No need for read-only transactions to contact the TSO Server
+ } else {
+ commitRegularTransaction(tx);
+ }
+ committedTxsCounter.inc();
+ } finally {
+ commitTimer.stop();
+ }
+
+ postCommit(tx);
+
+ } catch (TransactionManagerException e) {
+ throw new TransactionException(e.getMessage(), e);
+ }
+
+ }
+
+ /**
+ * Allows transaction manager developers to perform actions after committing a transaction.
+ * @param transaction
+ * the transaction that was committed.
+ * @throws TransactionManagerException
+ */
+ public void postCommit(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
+
+ /**
+ * Allows transaction manager developers to perform actions before rolling-back a transaction.
+ * @param transaction
+ * the transaction that is going to be rolled-back.
+ * @throws TransactionManagerException
+ */
+ public void preRollback(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
+
+ /**
+ * @see org.apache.omid.transaction.TransactionManager#rollback(Transaction)
+ */
+ @Override
+ public final void rollback(Transaction transaction) throws TransactionException {
+
+ AbstractTransaction<? extends CellId> tx = enforceAbstractTransactionAsParam(transaction);
+ enforceTransactionIsInRunningState(tx);
+
+ try {
+
+ preRollback(tx);
+
+ // Make sure its commit timestamp is 0, so the cleanup does the right job
+ tx.setCommitTimestamp(0);
+ tx.setStatus(Status.ROLLEDBACK);
+
+ postRollback(tx);
+
+ } catch (TransactionManagerException e) {
+ throw new TransactionException(e.getMessage(), e);
+ } finally {
+ tx.cleanup();
+ }
+
+ }
+
+ /**
+ * Allows transaction manager developers to perform actions after rolling-back a transaction.
+ * @param transaction
+ * the transaction that was rolled-back.
+ * @throws TransactionManagerException
+ */
+ public void postRollback(AbstractTransaction<? extends CellId> transaction) throws TransactionManagerException {}
+
+ /**
+ * Check if the transaction commit data is in the shadow cell
+ * @param cellStartTimestamp
+ * the transaction start timestamp
+ * locator
+ * the timestamp locator
+ * @throws IOException
+ */
+ Optional<CommitTimestamp> readCommitTimestampFromShadowCell(long cellStartTimestamp, CommitTimestampLocator locator)
+ throws IOException
+ {
+
+ Optional<CommitTimestamp> commitTS = Optional.absent();
+
+ Optional<Long> commitTimestamp = locator.readCommitTimestampFromShadowCell(cellStartTimestamp);
+ if (commitTimestamp.isPresent()) {
+ commitTS = Optional.of(new CommitTimestamp(SHADOW_CELL, commitTimestamp.get(), true)); // Valid commit TS
+ }
+
+ return commitTS;
+ }
+
+ /**
+ * This function returns the commit timestamp for a particular cell if the transaction was already committed in
+ * the system. In case the transaction was not committed and the cell was written by transaction initialized by a
+ * previous TSO server, an invalidation try occurs.
+ * Otherwise the function returns a value that indicates that the commit timestamp was not found.
+ * @param cellStartTimestamp
+ * start timestamp of the cell to locate the commit timestamp for.
+ * @param epoch
+ * the epoch of the TSO server the current tso client is working with.
+ * @param locator
+ * a locator to find the commit timestamp in the system.
+ * @return the commit timestamp joint with the location where it was found
+ * or an object indicating that it was not found in the system
+ * @throws IOException
+ */
+ public CommitTimestamp locateCellCommitTimestamp(long cellStartTimestamp, long epoch,
+ CommitTimestampLocator locator) throws IOException {
+
+ try {
+ // 1) First check the cache
+ Optional<Long> commitTimestamp = locator.readCommitTimestampFromCache(cellStartTimestamp);
+ if (commitTimestamp.isPresent()) { // Valid commit timestamp
+ return new CommitTimestamp(CACHE, commitTimestamp.get(), true);
+ }
+
+ // 2) Then check the commit table
+ // If the data was written at a previous epoch, check whether the transaction was invalidated
+ Optional<CommitTimestamp> commitTimeStamp = commitTableClient.getCommitTimestamp(cellStartTimestamp).get();
+ if (commitTimeStamp.isPresent()) {
+ return commitTimeStamp.get();
+ }
+
+ // 3) Read from shadow cell
+ commitTimeStamp = readCommitTimestampFromShadowCell(cellStartTimestamp, locator);
+ if (commitTimeStamp.isPresent()) {
+ return commitTimeStamp.get();
+ }
+
+ // 4) Check the epoch and invalidate the entry
+ // if the data was written by a transaction from a previous epoch (previous TSO)
+ if (cellStartTimestamp < epoch) {
+ boolean invalidated = commitTableClient.tryInvalidateTransaction(cellStartTimestamp).get();
+ if (invalidated) { // Invalid commit timestamp
+ return new CommitTimestamp(COMMIT_TABLE, CommitTable.INVALID_TRANSACTION_MARKER, false);
+ }
+ }
+
+ // 5) We did not manage to invalidate the transactions then check the commit table
+ commitTimeStamp = commitTableClient.getCommitTimestamp(cellStartTimestamp).get();
+ if (commitTimeStamp.isPresent()) {
+ return commitTimeStamp.get();
+ }
+
+ // 6) Read from shadow cell
+ commitTimeStamp = readCommitTimestampFromShadowCell(cellStartTimestamp, locator);
+ if (commitTimeStamp.isPresent()) {
+ return commitTimeStamp.get();
+ }
+
+ // *) Otherwise return not found
+ return new CommitTimestamp(NOT_PRESENT, -1L /** TODO Check if we should return this */, true);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while finding commit timestamp", e);
+ } catch (ExecutionException e) {
+ throw new IOException("Problem finding commit timestamp", e);
+ }
+
+ }
+
+ /**
+ * @see java.io.Closeable#close()
+ */
+ @Override
+ public final void close() throws IOException {
+
+ tsoClient.close();
+ commitTableClient.close();
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private void enforceTransactionIsInRunningState(Transaction transaction) {
+
+ if (transaction.getStatus() != Status.RUNNING) {
+ throw new IllegalArgumentException("Transaction was already " + transaction.getStatus());
+ }
+
+ }
+
+ @SuppressWarnings("unchecked")
+ // NOTE: We are sure that tx is not parametrized
+ private AbstractTransaction<? extends CellId> enforceAbstractTransactionAsParam(Transaction tx) {
+
+ if (tx instanceof AbstractTransaction) {
+ return (AbstractTransaction<? extends CellId>) tx;
+ } else {
+ throw new IllegalArgumentException(
+ "The transaction object passed is not an instance of AbstractTransaction");
+ }
+
+ }
+
+ private void markReadOnlyTransaction(AbstractTransaction<? extends CellId> readOnlyTx) {
+
+ readOnlyTx.setStatus(Status.COMMITTED_RO);
+
+ }
+
+ private void commitRegularTransaction(AbstractTransaction<? extends CellId> tx)
+ throws RollbackException, TransactionException
+ {
+
+ try {
+
+ long commitTs = tsoClient.commit(tx.getStartTimestamp(), tx.getWriteSet()).get();
+ certifyCommitForTx(tx, commitTs);
+ updateShadowCellsAndRemoveCommitTableEntry(tx, postCommitter);
+
+ } catch (ExecutionException e) {
+
+ if (e.getCause() instanceof AbortException) { // TSO reports Tx conflicts as AbortExceptions in the future
+ rollback(tx);
+ rolledbackTxsCounter.inc();
+ throw new RollbackException("Conflicts detected in tx writeset", e.getCause());
+ }
+
+ if (e.getCause() instanceof ServiceUnavailableException
+ ||
+ e.getCause() instanceof NewTSOException
+ ||
+ e.getCause() instanceof ConnectionException) {
+
+ errorTxsCounter.inc();
+ try {
+ LOG.warn("Can't contact the TSO for receiving outcome for Tx {}. Checking Commit Table...", tx);
+ // Check the commit table to find if the target TSO woke up in the meantime and added the commit
+ // TODO: Decide what we should we do if we can not contact the commit table
+ Optional<CommitTimestamp> commitTimestamp =
+ commitTableClient.getCommitTimestamp(tx.getStartTimestamp()).get();
+ if (commitTimestamp.isPresent()) {
+ if (commitTimestamp.get().isValid()) {
+ LOG.warn("{}: Valid commit TS found in Commit Table. Committing Tx...", tx);
+ certifyCommitForTx(tx, commitTimestamp.get().getValue());
+ postCommitter.updateShadowCells(tx); // But do NOT remove transaction from commit table
+ } else { // Probably another Tx in a new TSO Server invalidated this transaction
+ LOG.warn("{}: Invalidated commit TS found in Commit Table. Rolling-back...", tx);
+ rollback(tx);
+ throw new RollbackException(tx + " invalidated by other Tx started", e.getCause());
+ }
+ } else {
+ LOG.warn("{}: Trying to invalidate Tx proactively in Commit Table...", tx);
+ boolean invalidated = commitTableClient.tryInvalidateTransaction(tx.getStartTimestamp()).get();
+ if (invalidated) {
+ LOG.warn("{}: Invalidated proactively in Commit Table. Rolling-back Tx...", tx);
+ invalidatedTxsCounter.inc();
+ rollback(tx); // Rollback proactively cause it's likely that a new TSOServer is now master
+ throw new RollbackException(tx + " rolled-back precautionary", e.getCause());
+ } else {
+ LOG.warn("{}: Invalidation could NOT be completed. Re-checking Commit Table...", tx);
+ // TODO: Decide what we should we do if we can not contact the commit table
+ commitTimestamp = commitTableClient.getCommitTimestamp(tx.getStartTimestamp()).get();
+ if (commitTimestamp.isPresent() && commitTimestamp.get().isValid()) {
+ LOG.warn("{}: Valid commit TS found in Commit Table. Committing Tx...", tx);
+ certifyCommitForTx(tx, commitTimestamp.get().getValue());
+ postCommitter.updateShadowCells(tx); // But do NOT remove transaction from commit table
+ } else {
+ LOG.error("{}: Can't determine Transaction outcome", tx);
+ throw new TransactionException(tx + ": cannot determine Tx outcome");
+ }
+ }
+ }
+ } catch (ExecutionException e1) {
+ throw new TransactionException(tx + ": problem reading commitTS from Commit Table", e1);
+ } catch (InterruptedException e1) {
+ Thread.currentThread().interrupt();
+ throw new TransactionException(tx + ": interrupted while reading commitTS from Commit Table", e1);
+ }
+ } else {
+ throw new TransactionException(tx + ": cannot determine Tx outcome", e.getCause());
+ }
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ throw new TransactionException(tx + ": interrupted during commit", ie);
+
+ }
+
+ }
+
+ private void updateShadowCellsAndRemoveCommitTableEntry(final AbstractTransaction<? extends CellId> tx,
+ final PostCommitActions postCommitter) {
+
+ Futures.transform(postCommitter.updateShadowCells(tx), new Function<Void, Void>() {
+ @Override
+ public Void apply(Void aVoid) {
+ postCommitter.removeCommitTableEntry(tx);
+ return null;
+ }
+ });
+
+ }
+
+ private void certifyCommitForTx(AbstractTransaction<? extends CellId> txToSetup, long commitTS) {
+
+ txToSetup.setStatus(Status.COMMITTED);
+ txToSetup.setCommitTimestamp(commitTS);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/CommitTimestampLocator.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/CommitTimestampLocator.java b/transaction-client/src/main/java/org/apache/omid/transaction/CommitTimestampLocator.java
new file mode 100644
index 0000000..993c51e
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/CommitTimestampLocator.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+
+import java.io.IOException;
+
+/**
+ * An behavior that needs to be implemented by transaction managers
+ * to try to locate the possible commit timestamp for a cell. It
+ * allows to the transaction managers implementing this interface to
+ * search the commit timestamp in a local cache of their own or in
+ * their particular shadow cells implementation.
+ */
+public interface CommitTimestampLocator {
+
+ public Optional<Long> readCommitTimestampFromCache(long startTimestamp);
+
+ public Optional<Long> readCommitTimestampFromShadowCell(long startTimestamp)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/PostCommitActions.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/PostCommitActions.java b/transaction-client/src/main/java/org/apache/omid/transaction/PostCommitActions.java
new file mode 100644
index 0000000..0a06ede
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/PostCommitActions.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.omid.tso.client.CellId;
+
+public interface PostCommitActions {
+
+ /**
+ * Allows specific implementations to update the shadow cells.
+ * @param transaction
+ * the transaction to update shadow cells for
+ * @return future signalling end of the computation
+ */
+ ListenableFuture<Void> updateShadowCells(AbstractTransaction<? extends CellId> transaction);
+
+ /**
+ * Allows specific implementations to remove the transaction entry from the commit table.
+ * @param transaction
+ * the transaction to remove the commit table entry
+ * @return future signalling end of the computation
+ */
+ ListenableFuture<Void> removeCommitTableEntry(AbstractTransaction<? extends CellId> transaction);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/RollbackException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/RollbackException.java b/transaction-client/src/main/java/org/apache/omid/transaction/RollbackException.java
new file mode 100644
index 0000000..a82a7f8
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/RollbackException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+public class RollbackException extends Exception {
+
+ private static final long serialVersionUID = -9163407697376986830L;
+
+ public RollbackException(String message) {
+ super(message);
+ }
+
+ public RollbackException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public RollbackException() {
+ super();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/Transaction.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/Transaction.java b/transaction-client/src/main/java/org/apache/omid/transaction/Transaction.java
new file mode 100644
index 0000000..9e14f7a
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/Transaction.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Optional;
+
+/**
+ * This interface defines the transaction state & behavior exposed to users.
+ */
+public interface Transaction {
+
+ enum Status {
+ RUNNING, COMMITTED, ROLLEDBACK, COMMITTED_RO
+ }
+
+ /**
+ * Returns the transaction identifier
+ * @return the transaction identifier
+ */
+ long getTransactionId();
+
+ /**
+ * Returns the epoch given by the TSOServer
+ * @return the transaction's TSOServer epoch
+ */
+ long getEpoch();
+
+ /**
+ * Returns the current transaction {@link Status}
+ * @return transaction status
+ */
+ Status getStatus();
+
+ /**
+ * Forces the transaction to rollback, even when there's an intention
+ * to commit it.
+ */
+ void setRollbackOnly();
+
+ /**
+ * Returns whether the transaction was marked for rollback or not
+ * @return whether the transaction is marked for rollback or not
+ */
+ boolean isRollbackOnly();
+
+
+ /**
+ * Set of methods to attach some metadata to a transaction object. One example
+ * of such metadata are notifications
+ *
+ *
+ * Expects they metadata stored under key "key" to be of the "Set" type,
+ * append "value" to the existing set or creates a new one
+ */
+ void appendMetadata(String key, Object value);
+
+ void setMetadata(String key, Object value);
+
+ Optional<Object> getMetadata(String key);
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/TransactionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/TransactionException.java b/transaction-client/src/main/java/org/apache/omid/transaction/TransactionException.java
new file mode 100644
index 0000000..de75d1b
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/TransactionException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+public class TransactionException extends Exception {
+
+ private static final long serialVersionUID = 7273525983622126275L;
+
+ public TransactionException(String reason) {
+ super(reason);
+ }
+
+ public TransactionException(String reason, Throwable e) {
+ super(reason, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManager.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManager.java b/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManager.java
new file mode 100644
index 0000000..18de49b
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManager.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import java.io.Closeable;
+
+/**
+ * Provides the methods to manage transactions (create, commit...)
+ */
+public interface TransactionManager extends Closeable {
+
+ /**
+ * Starts a new transaction.
+ *
+ * Creates & returns a {@link Transaction} interface implementation
+ * that will be used in {@link TTable}'s methods for doing operations
+ * on the transactional context defined by the returned object.
+ *
+ * @return transaction
+ * representation of the created transaction
+ * @throws TransactionException
+ */
+ public Transaction begin() throws TransactionException;
+
+ /**
+ * Commits a transaction.
+ *
+ * If the transaction was marked for rollback or has conflicts
+ * with another concurrent transaction it will be rolledback
+ * automatically and a {@link RollbackException} will be thrown.
+ *
+ * @param tx
+ * transaction to be committed.
+ * @throws RollbackException
+ * thrown when transaction has conflicts with another transaction
+ * or when was marked for rollback.
+ * @throws TransactionException
+ */
+ public void commit(Transaction tx) throws RollbackException, TransactionException;
+
+ /**
+ * Aborts a transaction.
+ *
+ * Automatically rollbacks the changes performed by the transaction.
+ *
+ * @param tx
+ * transaction to be rolled-back.
+ */
+ public void rollback(Transaction tx) throws TransactionException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManagerException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManagerException.java b/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManagerException.java
new file mode 100644
index 0000000..8da121d
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/transaction/TransactionManagerException.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+public class TransactionManagerException extends Exception {
+
+ private static final long serialVersionUID = 6811947817962260774L;
+
+ public TransactionManagerException(String reason) {
+ super(reason);
+ }
+
+ public TransactionManagerException(String reason, Throwable e) {
+ super(reason, e);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/AbortException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/AbortException.java b/transaction-client/src/main/java/org/apache/omid/tso/client/AbortException.java
new file mode 100644
index 0000000..ec9fb75
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/AbortException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+/**
+ * Thrown when the TSO server has aborted a transaction
+ */
+public class AbortException extends Exception {
+
+ private static final long serialVersionUID = 1861474360100681040L;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/CellId.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/CellId.java b/transaction-client/src/main/java/org/apache/omid/tso/client/CellId.java
new file mode 100644
index 0000000..998c81a
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/CellId.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+public interface CellId {
+
+ public long getCellId();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/ClosingException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/ClosingException.java b/transaction-client/src/main/java/org/apache/omid/tso/client/ClosingException.java
new file mode 100644
index 0000000..0155356
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/ClosingException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+/**
+ * Thrown when an error is produced when performing the actions required
+ * to close the communication with the TSO server
+ */
+public class ClosingException extends Exception {
+
+ private static final long serialVersionUID = -5681694952053689884L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/ConnectionException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/ConnectionException.java b/transaction-client/src/main/java/org/apache/omid/tso/client/ConnectionException.java
new file mode 100644
index 0000000..b5ab76e
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/ConnectionException.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import java.io.IOException;
+
+/**
+ * Thrown when there are problems with the comm channel with the TSO server
+ * (e.g. when it is closed or disconnected)
+ */
+public class ConnectionException extends IOException {
+
+ private static final long serialVersionUID = -8489539195700267443L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/ForwardingTSOFuture.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/ForwardingTSOFuture.java b/transaction-client/src/main/java/org/apache/omid/tso/client/ForwardingTSOFuture.java
new file mode 100644
index 0000000..99c0878
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/ForwardingTSOFuture.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class ForwardingTSOFuture<T> implements TSOFuture<T> {
+ private final ListenableFuture<T> future;
+
+ public ForwardingTSOFuture(ListenableFuture<T> future) {
+ this.future = future;
+ }
+
+ @Override
+ public boolean cancel(boolean mayInterruptIfRunning) {
+ return future.cancel(mayInterruptIfRunning);
+ }
+
+ @Override
+ public boolean isCancelled() {
+ return future.isCancelled();
+ }
+
+ @Override
+ public boolean isDone() {
+ return future.isDone();
+ }
+
+ @Override
+ public T get() throws InterruptedException, ExecutionException {
+ return future.get();
+ }
+
+ @Override
+ public T get(long timeout, TimeUnit unit)
+ throws InterruptedException, ExecutionException, TimeoutException {
+ return future.get(timeout, unit);
+ }
+
+ @Override
+ public void addListener(Runnable listener, Executor executor) {
+ future.addListener(listener, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/HandshakeFailedException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/HandshakeFailedException.java b/transaction-client/src/main/java/org/apache/omid/tso/client/HandshakeFailedException.java
new file mode 100644
index 0000000..76589bb
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/HandshakeFailedException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+/**
+ * Thrown when some incompatibilities between the TSO client & server are
+ * found
+ */
+public class HandshakeFailedException extends Exception {
+
+ private static final long serialVersionUID = 8545505066920548834L;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/MockTSOClient.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/MockTSOClient.java b/transaction-client/src/main/java/org/apache/omid/tso/client/MockTSOClient.java
new file mode 100644
index 0000000..5df1567
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/MockTSOClient.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.omid.committable.CommitTable;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MockTSOClient implements TSOProtocol {
+ private final AtomicLong timestampGenerator = new AtomicLong();
+ private final int CONFLICT_MAP_SIZE = 1 * 1000 * 1000;
+ private final long[] conflictMap = new long[CONFLICT_MAP_SIZE];
+ private final AtomicLong lwm = new AtomicLong();
+
+ private final CommitTable.Writer commitTable;
+
+ public MockTSOClient(CommitTable.Writer commitTable) {
+ this.commitTable = commitTable;
+ }
+
+ @Override
+ public TSOFuture<Long> getNewStartTimestamp() {
+ synchronized (conflictMap) {
+ SettableFuture<Long> f = SettableFuture.<Long>create();
+ f.set(timestampGenerator.incrementAndGet());
+ return new ForwardingTSOFuture<Long>(f);
+ }
+ }
+
+ @Override
+ public TSOFuture<Long> commit(long transactionId, Set<? extends CellId> cells) {
+ synchronized (conflictMap) {
+ SettableFuture<Long> f = SettableFuture.<Long>create();
+ if (transactionId < lwm.get()) {
+ f.setException(new AbortException());
+ return new ForwardingTSOFuture<Long>(f);
+ }
+
+ boolean canCommit = true;
+ for (CellId c : cells) {
+ int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
+ if (conflictMap[index] >= transactionId) {
+ canCommit = false;
+ break;
+ }
+ }
+
+ if (canCommit) {
+ long commitTimestamp = timestampGenerator.incrementAndGet();
+ for (CellId c : cells) {
+ int index = Math.abs((int) (c.getCellId() % CONFLICT_MAP_SIZE));
+ long oldVal = conflictMap[index];
+ conflictMap[index] = commitTimestamp;
+ long curLwm = lwm.get();
+ while (oldVal > curLwm) {
+ if (lwm.compareAndSet(curLwm, oldVal)) {
+ break;
+ }
+ curLwm = lwm.get();
+ }
+ }
+
+ f.set(commitTimestamp);
+ try {
+ commitTable.addCommittedTransaction(transactionId, commitTimestamp);
+ commitTable.updateLowWatermark(lwm.get());
+ commitTable.flush();
+ } catch (IOException ioe) {
+ f.setException(ioe);
+ }
+ } else {
+ f.setException(new AbortException());
+ }
+ return new ForwardingTSOFuture<Long>(f);
+ }
+ }
+
+ @Override
+ public TSOFuture<Void> close() {
+ SettableFuture<Void> f = SettableFuture.<Void>create();
+ f.set(null);
+ return new ForwardingTSOFuture<Void>(f);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/NewTSOException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/NewTSOException.java b/transaction-client/src/main/java/org/apache/omid/tso/client/NewTSOException.java
new file mode 100644
index 0000000..799208e
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/NewTSOException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+/**
+ * Thrown when a new TSO has been detected
+ */
+public class NewTSOException extends Exception {
+
+ private static final long serialVersionUID = -3250655858200759321L;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/OmidClientConfiguration.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/OmidClientConfiguration.java b/transaction-client/src/main/java/org/apache/omid/tso/client/OmidClientConfiguration.java
new file mode 100644
index 0000000..f1dcfd0
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/OmidClientConfiguration.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+import org.apache.omid.YAMLUtils;
+
+/**
+ * Configuration for Omid client side
+ */
+public class OmidClientConfiguration {
+
+ private static final String DEFAULT_CONFIG_FILE_NAME = "omid-client-config.yml";
+
+ public enum ConnType {DIRECT, HA}
+
+ public enum PostCommitMode {SYNC, ASYNC}
+
+ // Basic connection related params
+
+ private ConnType connectionType = ConnType.DIRECT;
+ private String connectionString;
+ private String zkCurrentTsoPath;
+ private String zkNamespace;
+ private int zkConnectionTimeoutInSecs;
+
+ // Communication protocol related params
+
+ private int requestMaxRetries;
+ private int requestTimeoutInMs;
+ private int reconnectionDelayInSecs;
+ private int retryDelayInMs;
+ private int executorThreads;
+
+ // Transaction Manager related params
+
+ private PostCommitMode postCommitMode = PostCommitMode.SYNC;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Instantiation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public OmidClientConfiguration() {
+ new YAMLUtils().loadSettings(DEFAULT_CONFIG_FILE_NAME, this);
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Getters and setters for config params
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public ConnType getConnectionType() {
+ return connectionType;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.connectionType")
+ public void setConnectionType(ConnType connectionType) {
+ this.connectionType = connectionType;
+ }
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.connectionString")
+ public void setConnectionString(String connectionString) {
+ this.connectionString = connectionString;
+ }
+
+ public int getZkConnectionTimeoutInSecs() {
+ return zkConnectionTimeoutInSecs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.zkConnectionTimeoutInSecs")
+ public void setZkConnectionTimeoutInSecs(int zkConnectionTimeoutInSecs) {
+ this.zkConnectionTimeoutInSecs = zkConnectionTimeoutInSecs;
+ }
+
+ public int getRequestMaxRetries() {
+ return requestMaxRetries;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.requestMaxRetries")
+ public void setRequestMaxRetries(int requestMaxRetries) {
+ this.requestMaxRetries = requestMaxRetries;
+ }
+
+ public int getRequestTimeoutInMs() {
+ return requestTimeoutInMs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.requestTimeoutInMs")
+ public void setRequestTimeoutInMs(int requestTimeoutInMs) {
+ this.requestTimeoutInMs = requestTimeoutInMs;
+ }
+
+ public int getReconnectionDelayInSecs() {
+ return reconnectionDelayInSecs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.reconnectionDelayInSecs")
+ public void setReconnectionDelayInSecs(int reconnectionDelayInSecs) {
+ this.reconnectionDelayInSecs = reconnectionDelayInSecs;
+ }
+
+ public int getRetryDelayInMs() {
+ return retryDelayInMs;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.retryDelayInMs")
+ public void setRetryDelayInMs(int retryDelayInMs) {
+ this.retryDelayInMs = retryDelayInMs;
+ }
+
+ public int getExecutorThreads() {
+ return executorThreads;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.client.executorThreads")
+ public void setExecutorThreads(int executorThreads) {
+ this.executorThreads = executorThreads;
+ }
+
+ public String getZkCurrentTsoPath() {
+ return zkCurrentTsoPath;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.ha.zkCurrentTsoPath")
+ public void setZkCurrentTsoPath(String zkCurrentTsoPath) {
+ this.zkCurrentTsoPath = zkCurrentTsoPath;
+ }
+
+ public String getZkNamespace() {
+ return zkNamespace;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.ha.zkNamespace")
+ public void setZkNamespace(String zkNamespace) {
+ this.zkNamespace = zkNamespace;
+ }
+
+ public PostCommitMode getPostCommitMode() {
+ return postCommitMode;
+ }
+
+ @Inject(optional = true)
+ @Named("omid.tm.postCommitMode")
+ public void setPostCommitMode(PostCommitMode postCommitMode) {
+ this.postCommitMode = postCommitMode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/transaction-client/src/main/java/org/apache/omid/tso/client/ServiceUnavailableException.java
----------------------------------------------------------------------
diff --git a/transaction-client/src/main/java/org/apache/omid/tso/client/ServiceUnavailableException.java b/transaction-client/src/main/java/org/apache/omid/tso/client/ServiceUnavailableException.java
new file mode 100644
index 0000000..ea320f0
--- /dev/null
+++ b/transaction-client/src/main/java/org/apache/omid/tso/client/ServiceUnavailableException.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tso.client;
+
+/**
+ * Thrown when the requests from TSO client to the TSO server have reached
+ * a number of retries
+ */
+public class ServiceUnavailableException extends Exception {
+
+ private static final long serialVersionUID = -1551974284011474385L;
+
+ public ServiceUnavailableException(String message) {
+ super(message);
+ }
+
+}
[39/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/TTable.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/TTable.java b/hbase-client/src/main/java/org/apache/omid/transaction/TTable.java
new file mode 100644
index 0000000..6aec0b0
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/TTable.java
@@ -0,0 +1,770 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimaps;
+import org.apache.omid.committable.CommitTable.CommitTimestamp;
+import org.apache.omid.transaction.HBaseTransactionManager.CommitTimestampLocatorImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.TimeRange;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+
+/**
+ * Provides transactional methods for accessing and modifying a given snapshot
+ * of data identified by an opaque {@link Transaction} object. It mimics the
+ * behavior in {@link org.apache.hadoop.hbase.client.HTableInterface}
+ */
+public class TTable implements Closeable {
+
+ private static Logger LOG = LoggerFactory.getLogger(TTable.class);
+
+ private final HTableInterface healerTable;
+
+ private HTableInterface table;
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Construction
+ // ----------------------------------------------------------------------------------------------------------------
+
+ public TTable(Configuration conf, byte[] tableName) throws IOException {
+ this(new HTable(conf, tableName));
+ }
+
+ public TTable(String tableName) throws IOException {
+ this(HBaseConfiguration.create(), Bytes.toBytes(tableName));
+ }
+
+ public TTable(Configuration conf, String tableName) throws IOException {
+ this(conf, Bytes.toBytes(tableName));
+ }
+
+ public TTable(HTableInterface hTable) throws IOException {
+ table = hTable;
+ healerTable = new HTable(table.getConfiguration(), table.getTableName());
+ }
+
+ public TTable(HTableInterface hTable, HTableInterface healerTable) throws IOException {
+ table = hTable;
+ this.healerTable = healerTable;
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Closeable implementation
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Releases any resources held or pending changes in internal buffers.
+ *
+ * @throws IOException
+ * if a remote or network exception occurs.
+ */
+ @Override
+ public void close() throws IOException {
+ table.close();
+ healerTable.close();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Transactional operations
+ // ----------------------------------------------------------------------------------------------------------------
+
+ /**
+ * Transactional version of {@link HTableInterface#get(Get get)}
+ */
+ public Result get(Transaction tx, final Get get) throws IOException {
+
+ throwExceptionIfOpSetsTimerange(get);
+
+ HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
+
+ final long readTimestamp = transaction.getStartTimestamp();
+ final Get tsget = new Get(get.getRow()).setFilter(get.getFilter());
+ TimeRange timeRange = get.getTimeRange();
+ long startTime = timeRange.getMin();
+ long endTime = Math.min(timeRange.getMax(), readTimestamp + 1);
+ tsget.setTimeRange(startTime, endTime).setMaxVersions(1);
+ Map<byte[], NavigableSet<byte[]>> kvs = get.getFamilyMap();
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : kvs.entrySet()) {
+ byte[] family = entry.getKey();
+ NavigableSet<byte[]> qualifiers = entry.getValue();
+ if (qualifiers == null || qualifiers.isEmpty()) {
+ tsget.addFamily(family);
+ } else {
+ for (byte[] qualifier : qualifiers) {
+ tsget.addColumn(family, qualifier);
+ tsget.addColumn(family, CellUtils.addShadowCellSuffix(qualifier));
+ }
+ }
+ }
+ LOG.trace("Initial Get = {}", tsget);
+
+ // Return the KVs that belong to the transaction snapshot, ask for more
+ // versions if needed
+ Result result = table.get(tsget);
+ List<Cell> filteredKeyValues = Collections.emptyList();
+ if (!result.isEmpty()) {
+ filteredKeyValues = filterCellsForSnapshot(result.listCells(), transaction, tsget.getMaxVersions());
+ }
+
+ return Result.create(filteredKeyValues);
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#delete(Delete delete)}
+ */
+ public void delete(Transaction tx, Delete delete) throws IOException {
+
+ throwExceptionIfOpSetsTimerange(delete);
+
+ HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
+
+ final long startTimestamp = transaction.getStartTimestamp();
+ boolean issueGet = false;
+
+ final Put deleteP = new Put(delete.getRow(), startTimestamp);
+ final Get deleteG = new Get(delete.getRow());
+ Map<byte[], List<Cell>> fmap = delete.getFamilyCellMap();
+ if (fmap.isEmpty()) {
+ issueGet = true;
+ }
+ for (List<Cell> cells : fmap.values()) {
+ for (Cell cell : cells) {
+ CellUtils.validateCell(cell, startTimestamp);
+ switch (KeyValue.Type.codeToType(cell.getTypeByte())) {
+ case DeleteColumn:
+ deleteP.add(CellUtil.cloneFamily(cell),
+ CellUtil.cloneQualifier(cell),
+ startTimestamp,
+ CellUtils.DELETE_TOMBSTONE);
+ transaction.addWriteSetElement(
+ new HBaseCellId(table,
+ delete.getRow(),
+ CellUtil.cloneFamily(cell),
+ CellUtil.cloneQualifier(cell),
+ cell.getTimestamp()));
+ break;
+ case DeleteFamily:
+ deleteG.addFamily(CellUtil.cloneFamily(cell));
+ issueGet = true;
+ break;
+ case Delete:
+ if (cell.getTimestamp() == HConstants.LATEST_TIMESTAMP) {
+ deleteP.add(CellUtil.cloneFamily(cell),
+ CellUtil.cloneQualifier(cell),
+ startTimestamp,
+ CellUtils.DELETE_TOMBSTONE);
+ transaction.addWriteSetElement(
+ new HBaseCellId(table,
+ delete.getRow(),
+ CellUtil.cloneFamily(cell),
+ CellUtil.cloneQualifier(cell),
+ cell.getTimestamp()));
+ break;
+ } else {
+ throw new UnsupportedOperationException(
+ "Cannot delete specific versions on Snapshot Isolation.");
+ }
+ default:
+ break;
+ }
+ }
+ }
+ if (issueGet) {
+ // It's better to perform a transactional get to avoid deleting more
+ // than necessary
+ Result result = this.get(transaction, deleteG);
+ if (!result.isEmpty()) {
+ for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> entryF : result.getMap()
+ .entrySet()) {
+ byte[] family = entryF.getKey();
+ for (Entry<byte[], NavigableMap<Long, byte[]>> entryQ : entryF.getValue().entrySet()) {
+ byte[] qualifier = entryQ.getKey();
+ deleteP.add(family, qualifier, CellUtils.DELETE_TOMBSTONE);
+ transaction.addWriteSetElement(new HBaseCellId(table, delete.getRow(), family, qualifier, transaction.getStartTimestamp()));
+ }
+ }
+ }
+ }
+
+ if (!deleteP.isEmpty()) {
+ table.put(deleteP);
+ }
+
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#put(Put put)}
+ */
+ public void put(Transaction tx, Put put) throws IOException {
+
+ throwExceptionIfOpSetsTimerange(put);
+
+ HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
+
+ final long startTimestamp = transaction.getStartTimestamp();
+ // create put with correct ts
+ final Put tsput = new Put(put.getRow(), startTimestamp);
+ Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
+ for (List<Cell> kvl : kvs.values()) {
+ for (Cell c : kvl) {
+ CellUtils.validateCell(c, startTimestamp);
+ // Reach into keyvalue to update timestamp.
+ // It's not nice to reach into keyvalue internals,
+ // but we want to avoid having to copy the whole thing
+ KeyValue kv = KeyValueUtil.ensureKeyValue(c);
+ Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), startTimestamp);
+ tsput.add(kv);
+
+ transaction.addWriteSetElement(
+ new HBaseCellId(table,
+ CellUtil.cloneRow(kv),
+ CellUtil.cloneFamily(kv),
+ CellUtil.cloneQualifier(kv),
+ kv.getTimestamp()));
+ }
+ }
+
+ table.put(tsput);
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#getScanner(Scan scan)}
+ */
+ public ResultScanner getScanner(Transaction tx, Scan scan) throws IOException {
+
+ throwExceptionIfOpSetsTimerange(scan);
+
+ HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);
+
+ Scan tsscan = new Scan(scan);
+ tsscan.setMaxVersions(1);
+ tsscan.setTimeRange(0, transaction.getStartTimestamp() + 1);
+ Map<byte[], NavigableSet<byte[]>> kvs = scan.getFamilyMap();
+ for (Map.Entry<byte[], NavigableSet<byte[]>> entry : kvs.entrySet()) {
+ byte[] family = entry.getKey();
+ NavigableSet<byte[]> qualifiers = entry.getValue();
+ if (qualifiers == null) {
+ continue;
+ }
+ for (byte[] qualifier : qualifiers) {
+ tsscan.addColumn(family, CellUtils.addShadowCellSuffix(qualifier));
+ }
+ }
+ return new TransactionalClientScanner(transaction, tsscan, 1);
+ }
+
+ /**
+ * Filters the raw results returned from HBase and returns only those
+ * belonging to the current snapshot, as defined by the transaction
+ * object. If the raw results don't contain enough information for a
+ * particular qualifier, it will request more versions from HBase.
+ *
+ * @param rawCells
+ * Raw cells that we are going to filter
+ * @param transaction
+ * Defines the current snapshot
+ * @param versionsToRequest
+ * Number of versions requested from hbase
+ * @return Filtered KVs belonging to the transaction snapshot
+ * @throws IOException
+ */
+ List<Cell> filterCellsForSnapshot(List<Cell> rawCells, HBaseTransaction transaction,
+ int versionsToRequest) throws IOException {
+
+ assert (rawCells != null && transaction != null && versionsToRequest >= 1);
+
+ List<Cell> keyValuesInSnapshot = new ArrayList<>();
+ List<Get> pendingGetsList = new ArrayList<>();
+
+ int numberOfVersionsToFetch = versionsToRequest * 2;
+ if (numberOfVersionsToFetch < 1) {
+ numberOfVersionsToFetch = versionsToRequest;
+ }
+
+ Map<Long, Long> commitCache = buildCommitCache(rawCells);
+
+ for (Collection<Cell> columnCells : groupCellsByColumnFilteringShadowCells(rawCells)) {
+ boolean snapshotValueFound = false;
+ Cell oldestCell = null;
+ for (Cell cell : columnCells) {
+ if (isCellInSnapshot(cell, transaction, commitCache)) {
+ if (!CellUtil.matchingValue(cell, CellUtils.DELETE_TOMBSTONE)) {
+ keyValuesInSnapshot.add(cell);
+ }
+ snapshotValueFound = true;
+ break;
+ }
+ oldestCell = cell;
+ }
+ if (!snapshotValueFound) {
+ assert (oldestCell != null);
+ Get pendingGet = createPendingGet(oldestCell, numberOfVersionsToFetch);
+ pendingGetsList.add(pendingGet);
+ }
+ }
+
+ if (!pendingGetsList.isEmpty()) {
+ Result[] pendingGetsResults = table.get(pendingGetsList);
+ for (Result pendingGetResult : pendingGetsResults) {
+ if (!pendingGetResult.isEmpty()) {
+ keyValuesInSnapshot.addAll(
+ filterCellsForSnapshot(pendingGetResult.listCells(), transaction, numberOfVersionsToFetch));
+ }
+ }
+ }
+
+ Collections.sort(keyValuesInSnapshot, KeyValue.COMPARATOR);
+
+ assert (keyValuesInSnapshot.size() <= rawCells.size());
+ return keyValuesInSnapshot;
+ }
+
+ private Map<Long, Long> buildCommitCache(List<Cell> rawCells) {
+
+ Map<Long, Long> commitCache = new HashMap<>();
+
+ for (Cell cell : rawCells) {
+ if (CellUtils.isShadowCell(cell)) {
+ commitCache.put(cell.getTimestamp(), Bytes.toLong(CellUtil.cloneValue(cell)));
+ }
+ }
+
+ return commitCache;
+ }
+
+ private boolean isCellInSnapshot(Cell kv, HBaseTransaction transaction, Map<Long, Long> commitCache)
+ throws IOException {
+
+ long startTimestamp = transaction.getStartTimestamp();
+
+ if (kv.getTimestamp() == startTimestamp) {
+ return true;
+ }
+
+ Optional<Long> commitTimestamp =
+ tryToLocateCellCommitTimestamp(transaction.getTransactionManager(), transaction.getEpoch(), kv, commitCache);
+
+ return commitTimestamp.isPresent() && commitTimestamp.get() < startTimestamp;
+ }
+
+ private Get createPendingGet(Cell cell, int versionCount) throws IOException {
+
+ Get pendingGet = new Get(CellUtil.cloneRow(cell));
+ pendingGet.addColumn(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));
+ pendingGet.addColumn(CellUtil.cloneFamily(cell), CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength()));
+ pendingGet.setMaxVersions(versionCount);
+ pendingGet.setTimeRange(0, cell.getTimestamp());
+
+ return pendingGet;
+ }
+
+ private Optional<Long> tryToLocateCellCommitTimestamp(AbstractTransactionManager transactionManager,
+ long epoch,
+ Cell cell,
+ Map<Long, Long> commitCache)
+ throws IOException {
+
+ CommitTimestamp tentativeCommitTimestamp =
+ transactionManager.locateCellCommitTimestamp(
+ cell.getTimestamp(),
+ epoch,
+ new CommitTimestampLocatorImpl(
+ new HBaseCellId(table,
+ CellUtil.cloneRow(cell),
+ CellUtil.cloneFamily(cell),
+ CellUtil.cloneQualifier(cell),
+ cell.getTimestamp()),
+ commitCache));
+
+ // If transaction that added the cell was invalidated
+ if (!tentativeCommitTimestamp.isValid()) {
+ return Optional.absent();
+ }
+
+ switch (tentativeCommitTimestamp.getLocation()) {
+ case COMMIT_TABLE:
+ // If the commit timestamp is found in the persisted commit table,
+ // that means the writing process of the shadow cell in the post
+ // commit phase of the client probably failed, so we heal the shadow
+ // cell with the right commit timestamp for avoiding further reads to
+ // hit the storage
+ healShadowCell(cell, tentativeCommitTimestamp.getValue());
+ case CACHE:
+ case SHADOW_CELL:
+ return Optional.of(tentativeCommitTimestamp.getValue());
+ case NOT_PRESENT:
+ return Optional.absent();
+ default:
+ assert (false);
+ return Optional.absent();
+ }
+ }
+
+ void healShadowCell(Cell cell, long commitTimestamp) {
+ Put put = new Put(CellUtil.cloneRow(cell));
+ byte[] family = CellUtil.cloneFamily(cell);
+ byte[] shadowCellQualifier = CellUtils.addShadowCellSuffix(cell.getQualifierArray(),
+ cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ put.add(family, shadowCellQualifier, cell.getTimestamp(), Bytes.toBytes(commitTimestamp));
+ try {
+ healerTable.put(put);
+ } catch (IOException e) {
+ LOG.warn("Failed healing shadow cell for kv {}", cell, e);
+ }
+ }
+
+ protected class TransactionalClientScanner implements ResultScanner {
+ private HBaseTransaction state;
+ private ResultScanner innerScanner;
+ private int maxVersions;
+
+ TransactionalClientScanner(HBaseTransaction state, Scan scan, int maxVersions)
+ throws IOException {
+ this.state = state;
+ this.innerScanner = table.getScanner(scan);
+ this.maxVersions = maxVersions;
+ }
+
+
+ @Override
+ public Result next() throws IOException {
+ List<Cell> filteredResult = Collections.emptyList();
+ while (filteredResult.isEmpty()) {
+ Result result = innerScanner.next();
+ if (result == null) {
+ return null;
+ }
+ if (!result.isEmpty()) {
+ filteredResult = filterCellsForSnapshot(result.listCells(), state, maxVersions);
+ }
+ }
+ return Result.create(filteredResult);
+ }
+
+ // In principle no need to override, copied from super.next(int) to make
+ // sure it works even if super.next(int)
+ // changes its implementation
+ @Override
+ public Result[] next(int nbRows) throws IOException {
+ // Collect values to be returned here
+ ArrayList<Result> resultSets = new ArrayList<>(nbRows);
+ for (int i = 0; i < nbRows; i++) {
+ Result next = next();
+ if (next != null) {
+ resultSets.add(next);
+ } else {
+ break;
+ }
+ }
+ return resultSets.toArray(new Result[resultSets.size()]);
+ }
+
+ @Override
+ public void close() {
+ innerScanner.close();
+ }
+
+ @Override
+ public Iterator<Result> iterator() {
+ return new ResultIterator(this);
+ }
+
+ // ------------------------------------------------------------------------------------------------------------
+ // --------------------------------- Helper class for TransactionalClientScanner ------------------------------
+ // ------------------------------------------------------------------------------------------------------------
+
+ class ResultIterator implements Iterator<Result> {
+
+ TransactionalClientScanner scanner;
+ Result currentResult;
+
+ ResultIterator(TransactionalClientScanner scanner) {
+ try {
+ this.scanner = scanner;
+ currentResult = scanner.next();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return currentResult != null && !currentResult.isEmpty();
+ }
+
+ @Override
+ public Result next() {
+ try {
+ Result result = currentResult;
+ currentResult = scanner.next();
+ return result;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Not implemented");
+ }
+
+ }
+
+ }
+
+ /**
+ * Delegates to {@link HTable#getTableName()}
+ */
+ public byte[] getTableName() {
+ return table.getTableName();
+ }
+
+ /**
+ * Delegates to {@link HTable#getConfiguration()}
+ */
+ public Configuration getConfiguration() {
+ return table.getConfiguration();
+ }
+
+ /**
+ * Delegates to {@link HTable#getTableDescriptor()}
+ */
+ public HTableDescriptor getTableDescriptor() throws IOException {
+ return table.getTableDescriptor();
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#exists(Get get)}
+ */
+ public boolean exists(Transaction transaction, Get get) throws IOException {
+ Result result = get(transaction, get);
+ return !result.isEmpty();
+ }
+
+ /* TODO What should we do with this methods???
+ * @Override public void batch(Transaction transaction, List<? extends Row>
+ * actions, Object[] results) throws IOException, InterruptedException {}
+ *
+ * @Override public Object[] batch(Transaction transaction, List<? extends
+ * Row> actions) throws IOException, InterruptedException {}
+ *
+ * @Override public <R> void batchCallback(Transaction transaction, List<?
+ * extends Row> actions, Object[] results, Callback<R> callback) throws
+ * IOException, InterruptedException {}
+ *
+ * @Override public <R> Object[] batchCallback(List<? extends Row> actions,
+ * Callback<R> callback) throws IOException, InterruptedException {}
+ */
+
+ /**
+ * Transactional version of {@link HTableInterface#get(List<Get> gets)}
+ */
+ public Result[] get(Transaction transaction, List<Get> gets) throws IOException {
+ Result[] results = new Result[gets.size()];
+ int i = 0;
+ for (Get get : gets) {
+ results[i++] = get(transaction, get);
+ }
+ return results;
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#getScanner(byte[] family)}
+ */
+ public ResultScanner getScanner(Transaction transaction, byte[] family) throws IOException {
+ Scan scan = new Scan();
+ scan.addFamily(family);
+ return getScanner(transaction, scan);
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#getScanner(byte[] family, byte[] qualifier)}
+ */
+ public ResultScanner getScanner(Transaction transaction, byte[] family, byte[] qualifier)
+ throws IOException {
+ Scan scan = new Scan();
+ scan.addColumn(family, qualifier);
+ return getScanner(transaction, scan);
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#put(List<Put> puts)}
+ */
+ public void put(Transaction transaction, List<Put> puts) throws IOException {
+ for (Put put : puts) {
+ put(transaction, put);
+ }
+ }
+
+ /**
+ * Transactional version of {@link HTableInterface#delete(List<Delete> deletes)}
+ */
+ public void delete(Transaction transaction, List<Delete> deletes) throws IOException {
+ for (Delete delete : deletes) {
+ delete(transaction, delete);
+ }
+ }
+
+ /**
+ * Provides access to the underliying HTable in order to configure it or to
+ * perform unsafe (non-transactional) operations. The latter would break the
+ * transactional guarantees of the whole system.
+ *
+ * @return The underlying HTable object
+ */
+ public HTableInterface getHTable() {
+ return table;
+ }
+
+ /**
+ * Delegates to {@link HTable#setAutoFlush(boolean autoFlush)}
+ */
+ public void setAutoFlush(boolean autoFlush) {
+ table.setAutoFlush(autoFlush, true);
+ }
+
+ /**
+ * Delegates to {@link HTable#isAutoFlush()}
+ */
+ public boolean isAutoFlush() {
+ return table.isAutoFlush();
+ }
+
+ /**
+ * Delegates to {@link HTable.getWriteBufferSize()}
+ */
+ public long getWriteBufferSize() {
+ return table.getWriteBufferSize();
+ }
+
+ /**
+ * Delegates to {@link HTable.setWriteBufferSize()}
+ */
+ public void setWriteBufferSize(long writeBufferSize) throws IOException {
+ table.setWriteBufferSize(writeBufferSize);
+ }
+
+ /**
+ * Delegates to {@link HTable.flushCommits()}
+ */
+ public void flushCommits() throws IOException {
+ table.flushCommits();
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private void throwExceptionIfOpSetsTimerange(Get getOperation) {
+ TimeRange tr = getOperation.getTimeRange();
+ checkTimerangeIsSetToDefaultValuesOrThrowException(tr);
+ }
+
+ private void throwExceptionIfOpSetsTimerange(Scan scanOperation) {
+ TimeRange tr = scanOperation.getTimeRange();
+ checkTimerangeIsSetToDefaultValuesOrThrowException(tr);
+ }
+
+ private void checkTimerangeIsSetToDefaultValuesOrThrowException(TimeRange tr) {
+ if (tr.getMin() != 0L || tr.getMax() != Long.MAX_VALUE) {
+ throw new IllegalArgumentException(
+ "Timestamp/timerange not allowed in transactional user operations");
+ }
+ }
+
+ private void throwExceptionIfOpSetsTimerange(Mutation userOperation) {
+ if (userOperation.getTimeStamp() != HConstants.LATEST_TIMESTAMP) {
+ throw new IllegalArgumentException(
+ "Timestamp not allowed in transactional user operations");
+ }
+ }
+
+ private HBaseTransaction enforceHBaseTransactionAsParam(Transaction tx) {
+ if (tx instanceof HBaseTransaction) {
+ return (HBaseTransaction) tx;
+ } else {
+ throw new IllegalArgumentException(
+ String.format("The transaction object passed %s is not an instance of HBaseTransaction",
+ tx.getClass().getName()));
+ }
+ }
+
+ static ImmutableList<Collection<Cell>> groupCellsByColumnFilteringShadowCells(List<Cell> rawCells) {
+
+ Predicate<Cell> shadowCellFilter = new Predicate<Cell>() {
+
+ @Override
+ public boolean apply(Cell cell) {
+ return !CellUtils.isShadowCell(cell);
+ }
+
+ };
+
+ Function<Cell, ColumnWrapper> cellToColumnWrapper = new Function<Cell, ColumnWrapper>() {
+
+ @Override
+ public ColumnWrapper apply(Cell cell) {
+ return new ColumnWrapper(CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell));
+ }
+
+ };
+
+ return Multimaps.index(Iterables.filter(rawCells, shadowCellFilter), cellToColumnWrapper)
+ .asMap().values()
+ .asList();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/java/org/apache/omid/transaction/TTableCellGetterAdapter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/TTableCellGetterAdapter.java b/hbase-client/src/main/java/org/apache/omid/transaction/TTableCellGetterAdapter.java
new file mode 100644
index 0000000..600e901
--- /dev/null
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/TTableCellGetterAdapter.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.transaction;
+
+import org.apache.omid.transaction.CellUtils.CellGetter;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+
+import java.io.IOException;
+
+public class TTableCellGetterAdapter implements CellGetter {
+
+ private final TTable txTable;
+
+ public TTableCellGetterAdapter(TTable txTable) {
+ this.txTable = txTable;
+ }
+
+ public Result get(Get get) throws IOException {
+ return txTable.getHTable().get(get);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/resources/default-hbase-omid-client-config.yml b/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
index da59556..f2fd17a 100644
--- a/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
+++ b/hbase-client/src/main/resources/default-hbase-omid-client-config.yml
@@ -2,7 +2,7 @@
commitTableName: OMID_COMMIT_TABLE
#TSO/HA connection
-omidClientConfiguration: !!com.yahoo.omid.tso.client.OmidClientConfiguration [ ]
+omidClientConfiguration: !!org.apache.omid.tso.client.OmidClientConfiguration [ ]
#Instrumentation
-metrics: !!com.yahoo.omid.metrics.NullMetricsProvider [ ]
\ No newline at end of file
+metrics: !!org.apache.omid.metrics.NullMetricsProvider [ ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
deleted file mode 100644
index 74063b2..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/OmidTestBase.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.yahoo.omid.TestUtils;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.committable.InMemoryCommitTable;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig;
-import com.yahoo.omid.tools.hbase.OmidTableManager;
-import com.yahoo.omid.tso.TSOMockModule;
-import com.yahoo.omid.tso.TSOServer;
-import com.yahoo.omid.tso.TSOServerConfig;
-import com.yahoo.omid.tso.client.OmidClientConfiguration;
-import com.yahoo.omid.tso.client.TSOClient;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.AfterGroups;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeGroups;
-import org.testng.annotations.BeforeMethod;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-
-import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_RETRIES_NUMBER;
-
-public abstract class OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(OmidTestBase.class);
-
- static HBaseTestingUtility hBaseUtils;
- private static MiniHBaseCluster hbaseCluster;
- static Configuration hbaseConf;
-
- protected static final String TEST_TABLE = "test";
- protected static final String TEST_FAMILY = "data";
- static final String TEST_FAMILY2 = "data2";
- private HBaseCommitTableConfig hBaseCommitTableConfig;
-
- @BeforeMethod(alwaysRun = true)
- public void beforeClass(Method method) throws Exception {
- Thread.currentThread().setName("UnitTest-" + method.getName());
- }
-
-
- @BeforeGroups(groups = "sharedHBase")
- public void beforeGroups(ITestContext context) throws Exception {
- // TSO Setup
- TSOServerConfig tsoConfig = new TSOServerConfig();
- tsoConfig.setPort(1234);
- tsoConfig.setMaxItems(1000);
- Injector injector = Guice.createInjector(new TSOMockModule(tsoConfig));
- LOG.info("Starting TSO");
- TSOServer tso = injector.getInstance(TSOServer.class);
- hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);
- HBaseTimestampStorageConfig hBaseTimestampStorageConfig = injector.getInstance(HBaseTimestampStorageConfig.class);
- tso.startAndWait();
- TestUtils.waitForSocketListening("localhost", 1234, 100);
- LOG.info("Finished loading TSO");
- context.setAttribute("tso", tso);
-
- OmidClientConfiguration clientConf = new OmidClientConfiguration();
- clientConf.setConnectionString("localhost:1234");
- context.setAttribute("clientConf", clientConf);
-
- InMemoryCommitTable commitTable = (InMemoryCommitTable) injector.getInstance(CommitTable.class);
- context.setAttribute("commitTable", commitTable);
-
- // Create the associated Handler
- TSOClient client = TSOClient.newInstance(clientConf);
- context.setAttribute("client", client);
-
- // ------------------------------------------------------------------------------------------------------------
- // HBase setup
- // ------------------------------------------------------------------------------------------------------------
- LOG.info("Creating HBase minicluster");
- hbaseConf = HBaseConfiguration.create();
- hbaseConf.setInt("hbase.hregion.memstore.flush.size", 10_000 * 1024);
- hbaseConf.setInt("hbase.regionserver.nbreservationblocks", 1);
- hbaseConf.setInt(HBASE_CLIENT_RETRIES_NUMBER, 3);
-
- File tempFile = File.createTempFile("OmidTest", "");
- tempFile.deleteOnExit();
- hbaseConf.set("hbase.rootdir", tempFile.getAbsolutePath());
-
- hBaseUtils = new HBaseTestingUtility(hbaseConf);
- hbaseCluster = hBaseUtils.startMiniCluster(1);
- hBaseUtils.createTable(Bytes.toBytes(hBaseTimestampStorageConfig.getTableName()),
- new byte[][]{hBaseTimestampStorageConfig.getFamilyName().getBytes()},
- Integer.MAX_VALUE);
-
- createTestTable();
- createCommitTable();
-
- LOG.info("HBase minicluster is up");
- }
-
- private void createTestTable() throws IOException {
- HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
- HTableDescriptor test_table_desc = new HTableDescriptor(TableName.valueOf(TEST_TABLE));
- HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY);
- HColumnDescriptor datafam2 = new HColumnDescriptor(TEST_FAMILY2);
- datafam.setMaxVersions(Integer.MAX_VALUE);
- datafam2.setMaxVersions(Integer.MAX_VALUE);
- test_table_desc.addFamily(datafam);
- test_table_desc.addFamily(datafam2);
- admin.createTable(test_table_desc);
- }
-
- private void createCommitTable() throws IOException {
- String[] args = new String[]{OmidTableManager.COMMIT_TABLE_COMMAND_NAME, "-numRegions", "1"};
- OmidTableManager omidTableManager = new OmidTableManager(args);
- omidTableManager.executeActionsOnHBase(hbaseConf);
- }
-
-
- private TSOServer getTSO(ITestContext context) {
- return (TSOServer) context.getAttribute("tso");
- }
-
-
- TSOClient getClient(ITestContext context) {
- return (TSOClient) context.getAttribute("client");
- }
-
- InMemoryCommitTable getCommitTable(ITestContext context) {
- return (InMemoryCommitTable) context.getAttribute("commitTable");
- }
-
- protected TransactionManager newTransactionManager(ITestContext context) throws Exception {
- return newTransactionManager(context, getClient(context));
- }
-
- protected TransactionManager newTransactionManager(ITestContext context, PostCommitActions postCommitActions) throws Exception {
- HBaseOmidClientConfiguration clientConf = new HBaseOmidClientConfiguration();
- clientConf.setConnectionString("localhost:1234");
- clientConf.setHBaseConfiguration(hbaseConf);
- return HBaseTransactionManager.builder(clientConf)
- .postCommitter(postCommitActions)
- .commitTableClient(getCommitTable(context).getClient())
- .tsoClient(getClient(context)).build();
- }
-
- protected TransactionManager newTransactionManager(ITestContext context, TSOClient tsoClient) throws Exception {
- HBaseOmidClientConfiguration clientConf = new HBaseOmidClientConfiguration();
- clientConf.setConnectionString("localhost:1234");
- clientConf.setHBaseConfiguration(hbaseConf);
- return HBaseTransactionManager.builder(clientConf)
- .commitTableClient(getCommitTable(context).getClient())
- .tsoClient(tsoClient).build();
- }
-
- protected TransactionManager newTransactionManager(ITestContext context, CommitTable.Client commitTableClient)
- throws Exception {
- HBaseOmidClientConfiguration clientConf = new HBaseOmidClientConfiguration();
- clientConf.setConnectionString("localhost:1234");
- clientConf.setHBaseConfiguration(hbaseConf);
- return HBaseTransactionManager.builder(clientConf)
- .commitTableClient(commitTableClient)
- .tsoClient(getClient(context)).build();
- }
-
- @AfterGroups(groups = "sharedHBase")
- public void afterGroups(ITestContext context) throws Exception {
- LOG.info("Tearing down OmidTestBase...");
- if (hbaseCluster != null) {
- hBaseUtils.shutdownMiniCluster();
- }
-
- getClient(context).close().get();
- getTSO(context).stopAndWait();
- TestUtils.waitForSocketNotListening("localhost", 1234, 1000);
- }
-
- @AfterMethod(groups = "sharedHBase", timeOut = 60_000)
- public void afterMethod() {
- try {
- LOG.info("tearing Down");
- HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
- deleteTable(admin, TableName.valueOf(TEST_TABLE));
- createTestTable();
- deleteTable(admin, TableName.valueOf(hBaseCommitTableConfig.getTableName()));
- createCommitTable();
- } catch (Exception e) {
- LOG.error("Error tearing down", e);
- }
- }
-
- void deleteTable(HBaseAdmin admin, TableName tableName) throws IOException {
- if (admin.tableExists(tableName)) {
- if (admin.isTableDisabled(tableName)) {
- admin.deleteTable(tableName);
- } else {
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
- }
- }
- }
-
- static boolean verifyValue(byte[] tableName, byte[] row,
- byte[] fam, byte[] col, byte[] value) {
-
- try (HTable table = new HTable(hbaseConf, tableName)) {
- Get g = new Get(row).setMaxVersions(1);
- Result r = table.get(g);
- Cell cell = r.getColumnLatestCell(fam, col);
-
- if (LOG.isTraceEnabled()) {
- LOG.trace("Value for " + Bytes.toString(tableName) + ":"
- + Bytes.toString(row) + ":" + Bytes.toString(fam)
- + Bytes.toString(col) + "=>" + Bytes.toString(CellUtil.cloneValue(cell))
- + " (" + Bytes.toString(value) + " expected)");
- }
-
- return Bytes.equals(CellUtil.cloneValue(cell), value);
- } catch (IOException e) {
- LOG.error("Error reading row " + Bytes.toString(tableName) + ":"
- + Bytes.toString(row) + ":" + Bytes.toString(fam)
- + Bytes.toString(col), e);
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
deleted file mode 100644
index ad3fad1..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAsynchronousPostCommitter.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.common.util.concurrent.SettableFuture;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.yahoo.omid.committable.CommitTable;
-import com.yahoo.omid.metrics.NullMetricsProvider;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestAsynchronousPostCommitter extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestAsynchronousPostCommitter.class);
-
- private static final byte[] family = Bytes.toBytes(TEST_FAMILY);
- private static final byte[] nonExistentFamily = Bytes.toBytes("non-existent");
- private static final byte[] qualifier = Bytes.toBytes("test-qual");
-
- byte[] row1 = Bytes.toBytes("test-is-committed1");
- byte[] row2 = Bytes.toBytes("test-is-committed2");
-
- @Test(timeOut = 30_000)
- public void testPostCommitActionsAreCalledAsynchronously(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = getCommitTable(context).getClient();
-
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- ListeningExecutorService postCommitExecutor =
- MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
- PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
-
- TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
-
- final CountDownLatch beforeUpdatingShadowCellsLatch = new CountDownLatch(1);
- final CountDownLatch afterUpdatingShadowCellsLatch = new CountDownLatch(1);
- final CountDownLatch beforeRemovingCTEntryLatch = new CountDownLatch(1);
- final CountDownLatch afterRemovingCTEntryLatch = new CountDownLatch(1);
-
- doAnswer(new Answer<ListenableFuture<Void>>() {
- public ListenableFuture<Void> answer(InvocationOnMock invocation) {
- try {
- beforeUpdatingShadowCellsLatch.await();
- invocation.callRealMethod();
- afterUpdatingShadowCellsLatch.countDown();
- } catch (Throwable throwable) {
- throwable.printStackTrace();
- }
- return SettableFuture.create();
- }
- }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
-
- doAnswer(new Answer<ListenableFuture<Void>>() {
- public ListenableFuture<Void> answer(InvocationOnMock invocation) {
- try {
- beforeRemovingCTEntryLatch.await();
- LOG.info("We are here");
- invocation.callRealMethod();
- afterRemovingCTEntryLatch.countDown();
- } catch (Throwable throwable) {
- throwable.printStackTrace();
- }
- return SettableFuture.create();
- }
- }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Execute tx with async post commit actions
- Transaction tx1 = tm.begin();
-
- Put put1 = new Put(row1);
- put1.add(family, qualifier, Bytes.toBytes("hey!"));
- txTable.put(tx1, put1);
- Put put2 = new Put(row2);
- put2.add(family, qualifier, Bytes.toBytes("hou!"));
- txTable.put(tx1, put2);
-
- tm.commit(tx1);
-
- long tx1Id = tx1.getTransactionId();
-
- // As we have paused the update of shadow cells, the shadow cells shouldn't be there yet
- assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
-
- // Commit Table should contain an entry for the transaction
- Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
- assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
-
- // Read from row1 and row2 in a different Tx and check that result is the data written by tx1 despite the
- // post commit actions have not been executed yet (the shadow cells healing process should make its work)
- Transaction tx2 = tm.begin();
- Get get1 = new Get(row1);
- Result result = txTable.get(tx2, get1);
- byte[] value = result.getValue(family, qualifier);
- assertNotNull(value);
- assertEquals("hey!", Bytes.toString(value));
-
- Get get2 = new Get(row2);
- result = txTable.get(tx2, get2);
- value = result.getValue(family, qualifier);
- assertNotNull(value);
- assertEquals("hou!", Bytes.toString(value));
-
- // Then, we continue with the update of shadow cells and we wait till completed
- beforeUpdatingShadowCellsLatch.countDown();
- afterUpdatingShadowCellsLatch.await();
-
- // Now we can check that the shadow cells are there...
- verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
- assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- // ...and the transaction entry is still in the Commit Table
- commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
- assertEquals(commitTimestamp.get().getValue(), ((AbstractTransaction) tx1).getCommitTimestamp());
-
- // Finally, we continue till the Commit Table cleaning process is done...
- beforeRemovingCTEntryLatch.countDown();
- afterRemovingCTEntryLatch.await();
-
- // ...so now, the Commit Table should NOT contain the entry for the transaction anymore
- verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
- commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
- assertFalse(commitTimestamp.isPresent());
-
- // Final checks
- verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
- verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
-
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testNoAsyncPostActionsAreCalled(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = getCommitTable(context).getClient();
-
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- ListeningExecutorService postCommitExecutor =
- MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
- PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
-
- TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
-
- final CountDownLatch updateShadowCellsCalledLatch = new CountDownLatch(1);
- final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
-
- // Simulate shadow cells are not updated and commit table is not clean
- doAnswer(new Answer<Void>() {
- public Void answer(InvocationOnMock invocation) {
- // Do not invoke real method simulating a fail of the shadow cells update
- updateShadowCellsCalledLatch.countDown();
- return null;
- }
- }).when(syncPostCommitter).updateShadowCells(any(AbstractTransaction.class));
-
- doAnswer(new Answer<Void>() {
- public Void answer(InvocationOnMock invocation) {
- // Do not invoke real method simulating a fail of the async clean of commit table entry
- removeCommitTableEntryCalledLatch.countDown();
- return null;
- }
- }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
-
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Execute tx with async post commit actions
- Transaction tx1 = tm.begin();
-
- Put put1 = new Put(row1);
- put1.add(family, qualifier, Bytes.toBytes("hey!"));
- txTable.put(tx1, put1);
- Put put2 = new Put(row2);
- put2.add(family, qualifier, Bytes.toBytes("hou!"));
- txTable.put(tx1, put2);
-
- tm.commit(tx1);
-
- long tx1Id = tx1.getTransactionId();
-
- // The shadow cells shouldn't be there...
- assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- // ... and the should NOT have been cleaned
- Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
-
- updateShadowCellsCalledLatch.await();
-
- // Not even after waiting for the method call on the shadow cells update...
- assertFalse(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- assertFalse(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
-
- removeCommitTableEntryCalledLatch.await();
- // ... and after waiting for the method call for cleaning the commit table entry
- commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
-
- // Final checks
- verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
- verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
-
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testOnlyShadowCellsUpdateIsExecuted(ITestContext context) throws Exception {
-
- CommitTable.Client commitTableClient = getCommitTable(context).getClient();
-
- PostCommitActions syncPostCommitter =
- spy(new HBaseSyncPostCommitter(new NullMetricsProvider(), commitTableClient));
- ListeningExecutorService postCommitExecutor =
- MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(
- new ThreadFactoryBuilder().setNameFormat("postCommit-%d").build()));
- PostCommitActions asyncPostCommitter = new HBaseAsyncPostCommitter(syncPostCommitter, postCommitExecutor);
-
- TransactionManager tm = newTransactionManager(context, asyncPostCommitter);
-
- final CountDownLatch removeCommitTableEntryCalledLatch = new CountDownLatch(1);
-
- doAnswer(new Answer<Void>() {
- public Void answer(InvocationOnMock invocation) {
- // Do not invoke real method simulating a fail of the async clean of commit table entry
- removeCommitTableEntryCalledLatch.countDown();
- return null;
- }
- }).when(syncPostCommitter).removeCommitTableEntry(any(AbstractTransaction.class));
-
-
- try (TTable txTable = new TTable(hbaseConf, TEST_TABLE)) {
-
- // Execute tx with async post commit actions
- Transaction tx1 = tm.begin();
-
- Put put1 = new Put(row1);
- put1.add(family, qualifier, Bytes.toBytes("hey!"));
- txTable.put(tx1, put1);
- Put put2 = new Put(row2);
- put2.add(family, qualifier, Bytes.toBytes("hou!"));
- txTable.put(tx1, put2);
-
- tm.commit(tx1);
-
- long tx1Id = tx1.getTransactionId();
-
- // We continue when the unsuccessful call of the method for cleaning commit table has been invoked
- removeCommitTableEntryCalledLatch.await();
-
- // We check that the shadow cells are there (because the update of the shadow cells should precede
- // the cleaning of the commit table entry) ...
- assertTrue(CellUtils.hasShadowCell(row1, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
- assertTrue(CellUtils.hasShadowCell(row2, family, qualifier, tx1Id, new TTableCellGetterAdapter(txTable)));
-
- // ... and the commit table entry has NOT been cleaned
- Optional<CommitTable.CommitTimestamp> commitTimestamp = commitTableClient.getCommitTimestamp(tx1Id).get();
- assertTrue(commitTimestamp.isPresent());
- assertTrue(commitTimestamp.get().isValid());
-
- // Final checks
- verify(syncPostCommitter, times(1)).updateShadowCells(any(AbstractTransaction.class));
- verify(syncPostCommitter, times(1)).removeCommitTableEntry(any(AbstractTransaction.class));
-
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
deleted file mode 100644
index c40d56d..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestAutoFlush.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.testng.AssertJUnit.assertEquals;
-
-@Test(groups = "sharedHBase")
-public class TestAutoFlush extends OmidTestBase {
-
- @Test
- public void testReadWithSeveralUncommitted(ITestContext context) throws Exception {
- byte[] family = Bytes.toBytes(TEST_FAMILY);
- byte[] row = Bytes.toBytes("row");
- byte[] col = Bytes.toBytes("col1");
- byte[] data = Bytes.toBytes("data");
- TransactionManager tm = newTransactionManager(context);
- TTable table = new TTable(hbaseConf, TEST_TABLE);
-
- // Turn off autoflush
- table.setAutoFlush(false);
-
- Transaction t = tm.begin();
- Put put = new Put(row);
- put.add(family, col, data);
- table.put(t, put);
-
- // Data shouldn't be in DB yet
- Get get = new Get(row);
- Result result = table.getHTable().get(get);
- assertEquals("Writes are already in DB", 0, result.size());
-
- tm.commit(t);
-
- // After commit, both the cell and shadow cell should be there.
- // That's why we check for two elements in the test assertion
- result = table.getHTable().get(get);
- assertEquals("Writes were not flushed to DB", 2, result.size());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java b/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
deleted file mode 100644
index 65896cc..0000000
--- a/hbase-client/src/test/java/com/yahoo/omid/transaction/TestBasicTransaction.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.transaction;
-
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.ITestContext;
-import org.testng.annotations.Test;
-
-import static org.junit.Assert.fail;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-@Test(groups = "sharedHBase")
-public class TestBasicTransaction extends OmidTestBase {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestBasicTransaction.class);
-
-
- @Test(timeOut = 30_000)
- public void testTimestampsOfTwoRowsInstertedAfterCommitOfSingleTransactionAreEquals(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- byte[] rowName1 = Bytes.toBytes("row1");
- byte[] rowName2 = Bytes.toBytes("row2");
- byte[] famName1 = Bytes.toBytes(TEST_FAMILY);
- byte[] colName1 = Bytes.toBytes("col1");
- byte[] dataValue1 = Bytes.toBytes("testWrite-1");
- byte[] dataValue2 = Bytes.toBytes("testWrite-2");
-
- Transaction tx1 = tm.begin();
-
- Put row1 = new Put(rowName1);
- row1.add(famName1, colName1, dataValue1);
- tt.put(tx1, row1);
- Put row2 = new Put(rowName2);
- row2.add(famName1, colName1, dataValue2);
- tt.put(tx1, row2);
-
- tm.commit(tx1);
-
- tt.close();
-
- // Checks
- Get getResultRow1 = new Get(rowName1).setMaxVersions(1);
- Result result1 = tt.getHTable().get(getResultRow1);
- byte[] val1 = result1.getValue(famName1, colName1);
- assertTrue(Bytes.equals(dataValue1, result1.getValue(famName1, colName1)),
- "Unexpected value for row 1 in col 1: " + Bytes.toString(val1));
- long tsRow1 = result1.rawCells()[0].getTimestamp();
-
- Get getResultRow2 = new Get(rowName2).setMaxVersions(1);
- Result result2 = tt.getHTable().get(getResultRow2);
- byte[] val2 = result2.getValue(famName1, colName1);
- assertTrue(Bytes.equals(dataValue2, result2.getValue(famName1, colName1)),
- "Unexpected value for row 2 in col 1: " + Bytes.toString(val2));
- long tsRow2 = result2.rawCells()[0].getTimestamp();
-
- assertEquals(tsRow2, tsRow1, "Timestamps of row 1 and row 2 are different");
-
- }
-
- @Test(timeOut = 30_000)
- public void testTimestampsOfTwoRowsModifiedByTwoSequentialTransactionsAreEqualAndHaveBeenIncreasedMonotonically(ITestContext context)
- throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- byte[] rowName1 = Bytes.toBytes("row1");
- byte[] rowName2 = Bytes.toBytes("row2");
- byte[] famName1 = Bytes.toBytes(TEST_FAMILY);
- byte[] colName1 = Bytes.toBytes("col1");
- byte[] dataValue1 = Bytes.toBytes("testWrite-1");
- byte[] dataValue2 = Bytes.toBytes("testWrite-2");
-
- byte[] dataValue3 = Bytes.toBytes("testWrite-3");
- byte[] dataValue4 = Bytes.toBytes("testWrite-4");
-
- Transaction tx1 = tm.begin();
-
- Put row1 = new Put(rowName1);
- row1.add(famName1, colName1, dataValue1);
- tt.put(tx1, row1);
- Put row2 = new Put(rowName2);
- row2.add(famName1, colName1, dataValue2);
- tt.put(tx1, row2);
-
- tm.commit(tx1);
-
- Transaction tx2 = tm.begin();
-
- row1 = new Put(rowName1);
- row1.add(famName1, colName1, dataValue3);
- tt.put(tx2, row1);
- row2 = new Put(rowName2);
- row2.add(famName1, colName1, dataValue4);
- tt.put(tx2, row2);
-
- tm.commit(tx2);
-
- tt.close();
-
- // Checks
- Get getResultRow1 = new Get(rowName1).setMaxVersions(2);
- Result result1 = tt.getHTable().get(getResultRow1);
- byte[] val1 = result1.getValue(famName1, colName1);
- assertTrue(Bytes.equals(dataValue3, result1.getValue(famName1, colName1)),
- "Unexpected value for row 1 in col 1: " + Bytes.toString(val1));
-
- long lastTsRow1 = result1.rawCells()[0].getTimestamp();
- long previousTsRow1 = result1.rawCells()[1].getTimestamp();
-
- Get getResultRow2 = new Get(rowName2).setMaxVersions(2);
- Result result2 = tt.getHTable().get(getResultRow2);
- byte[] val2 = result2.getValue(famName1, colName1);
- assertTrue(Bytes.equals(dataValue4, result2.getValue(famName1, colName1)),
- "Unexpected value for row 2 in col 1: " + Bytes.toString(val2));
-
- long lastTsRow2 = result2.rawCells()[0].getTimestamp();
- long previousTsRow2 = result2.rawCells()[1].getTimestamp();
-
- assertTrue(lastTsRow1 == lastTsRow2, "Timestamps assigned by Tx2 to row 1 and row 2 are different");
- assertTrue(previousTsRow1 == previousTsRow2, "Timestamps assigned by Tx2 to row 1 and row 2 are different");
- assertTrue(lastTsRow1 > previousTsRow1, "Timestamp assigned by Tx2 to row 1 hasn't increased monotonically");
- assertTrue(lastTsRow2 > previousTsRow2, "Timestamp assigned by Tx2 to row 2 hasn't increased monotonically");
-
- }
-
- @Test(timeOut = 30_000)
- public void runTestSimple(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
-
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- byte[] row = Bytes.toBytes("test-simple");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
- tm.commit(t1);
-
- Transaction tread = tm.begin();
- Transaction t2 = tm.begin();
- p = new Put(row);
- p.add(fam, col, data2);
- tt.put(t2, p);
- tm.commit(t2);
-
- Get g = new Get(row).setMaxVersions(1);
- Result r = tt.getHTable().get(g);
- assertTrue(Bytes.equals(data2, r.getValue(fam, col)),
- "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
-
- r = tt.get(tread, g);
- assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
- "Unexpected value for SI read " + tread + ": " + Bytes.toString(r.getValue(fam, col)));
- }
-
- @Test(timeOut = 30_000)
- public void runTestManyVersions(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- byte[] row = Bytes.toBytes("test-simple");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
- tm.commit(t1);
-
- for (int i = 0; i < 5; ++i) {
- Transaction t2 = tm.begin();
- p = new Put(row);
- p.add(fam, col, data2);
- tt.put(t2, p);
- }
- Transaction tread = tm.begin();
-
- Get g = new Get(row).setMaxVersions(1);
- Result r = tt.getHTable().get(g);
- assertTrue(Bytes.equals(data2, r.getValue(fam, col)),
- "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
-
- r = tt.get(tread, g);
- assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
- "Unexpected value for SI read " + tread + ": " + Bytes.toString(r.getValue(fam, col)));
-
- }
-
- @Test(timeOut = 30_000)
- public void runTestInterleave(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable tt = new TTable(hbaseConf, TEST_TABLE);
-
- Transaction t1 = tm.begin();
- LOG.info("Transaction created " + t1);
-
- byte[] row = Bytes.toBytes("test-interleave");
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("testdata");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- tt.put(t1, p);
- tm.commit(t1);
-
- Transaction t2 = tm.begin();
- p = new Put(row);
- p.add(fam, col, data2);
- tt.put(t2, p);
-
- Transaction tread = tm.begin();
- Get g = new Get(row).setMaxVersions(1);
- Result r = tt.get(tread, g);
- assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
- "Unexpected value for SI read " + tread + ": " + Bytes.toString(r.getValue(fam, col)));
- tm.commit(t2);
-
- r = tt.getHTable().get(g);
- assertTrue(Bytes.equals(data2, r.getValue(fam, col)),
- "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
-
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, timeOut = 30_000)
- public void testSameCommitRaisesException(ITestContext context) throws Exception {
- TransactionManager tm = newTransactionManager(context);
-
- Transaction t1 = tm.begin();
- tm.commit(t1);
- tm.commit(t1);
- }
-
- @Test(timeOut = 30_000)
- public void testInterleavedScanReturnsTheRightSnapshotResults(ITestContext context) throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // Basic data-scaffolding for test
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("TEST_COL");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- byte[] startRow = Bytes.toBytes("row-to-scan" + 0);
- byte[] stopRow = Bytes.toBytes("row-to-scan" + 9);
- byte[] randomRow = Bytes.toBytes("row-to-scan" + 3);
-
- // Add some data transactionally to have an initial state for the test
- Transaction tx1 = tm.begin();
- for (int i = 0; i < 10; i++) {
- byte[] row = Bytes.toBytes("row-to-scan" + i);
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- txTable.put(tx1, p);
- }
- tm.commit(tx1);
-
- // Start a second transaction -Tx2- modifying a random row and check that a concurrent transactional context
- // that scans the table, gets the proper snapshot with the stuff written by Tx1
- Transaction tx2 = tm.begin();
- Put p = new Put(randomRow);
- p.add(fam, col, data2);
- txTable.put(tx2, p);
-
- Transaction scanTx = tm.begin(); // This is the concurrent transactional scanner
- ResultScanner rs = txTable.getScanner(scanTx, new Scan().setStartRow(startRow).setStopRow(stopRow));
- Result r = rs.next(); // Exercise the next() method
- int i = 0;
- while (r != null) {
- LOG.trace("Scan (" + ++i + ")" + Bytes.toString(r.getRow()) + " => " + Bytes.toString(r.getValue(fam, col)));
- assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
- "Unexpected value for SI scan " + scanTx + ": " + Bytes.toString(r.getValue(fam, col)));
- r = rs.next();
- }
-
- // Commit the Tx2 and then check that under a new transactional context, the scanner gets the right snapshot,
- // which must include the row modified by Tx2
- tm.commit(tx2);
-
- int modifiedRows = 0;
- Transaction newScanTx = tm.begin();
- ResultScanner newRS = txTable.getScanner(newScanTx, new Scan().setStartRow(startRow).setStopRow(stopRow));
- Result[] results = newRS.next(10); // Exercise the next(numRows) method
- for (Result result : results) {
- if (Bytes.equals(data2, result.getValue(fam, col))) {
- LOG.trace("Modified :" + Bytes.toString(result.getRow()));
- modifiedRows++;
- }
- }
- assertEquals(modifiedRows, 1, "Expected 1 row modified, but " + modifiedRows + " are.");
-
- // Same check as before but checking that the results are correct when retrieved through the Scanner Iterator
- modifiedRows = 0;
- ResultScanner iterableRS = txTable.getScanner(newScanTx, new Scan().setStartRow(startRow).setStopRow(stopRow));
- for (Result res : iterableRS) {
- if (Bytes.equals(data2, res.getValue(fam, col))) {
- LOG.trace("Modified :" + Bytes.toString(res.getRow()));
- modifiedRows++;
- }
- }
-
- assertEquals(modifiedRows, 1, "Expected 1 row modified, but " + modifiedRows + " are.");
-
- // Finally, check that the Scanner Iterator does not implement the remove method
- try {
- iterableRS.iterator().remove();
- fail();
- } catch (RuntimeException re) {
- // Expected
- }
-
- }
-
- @Test(timeOut = 30_000)
- public void testInterleavedScanReturnsTheRightSnapshotResultsWhenATransactionAborts(ITestContext context)
- throws Exception {
-
- TransactionManager tm = newTransactionManager(context);
- TTable txTable = new TTable(hbaseConf, TEST_TABLE);
-
- // Basic data-scaffolding for test
- byte[] fam = Bytes.toBytes(TEST_FAMILY);
- byte[] col = Bytes.toBytes("TEST_COL");
- byte[] data1 = Bytes.toBytes("testWrite-1");
- byte[] data2 = Bytes.toBytes("testWrite-2");
-
- byte[] startRow = Bytes.toBytes("row-to-scan" + 0);
- byte[] stopRow = Bytes.toBytes("row-to-scan" + 9);
- byte[] randomRow = Bytes.toBytes("row-to-scan" + 3);
-
- // Add some data transactionally to have an initial state for the test
- Transaction tx1 = tm.begin();
- for (int i = 0; i < 10; i++) {
- byte[] row = Bytes.toBytes("row-to-scan" + i);
-
- Put p = new Put(row);
- p.add(fam, col, data1);
- txTable.put(tx1, p);
- }
- tm.commit(tx1);
-
- // Start a second transaction modifying a random row and check that a transactional scanner in Tx2 gets the
- // right snapshot with the new value in the random row just written by Tx2
- Transaction tx2 = tm.begin();
- Put p = new Put(randomRow);
- p.add(fam, col, data2);
- txTable.put(tx2, p);
-
- int modifiedRows = 0;
- ResultScanner rs = txTable.getScanner(tx2, new Scan().setStartRow(startRow).setStopRow(stopRow));
- Result r = rs.next();
- while (r != null) {
- if (Bytes.equals(data2, r.getValue(fam, col))) {
- LOG.trace("Modified :" + Bytes.toString(r.getRow()));
- modifiedRows++;
- }
-
- r = rs.next();
- }
-
- assertEquals(modifiedRows, 1, "Expected 1 row modified, but " + modifiedRows + " are.");
-
- // Rollback the second transaction and then check that under a new transactional scanner we get the snapshot
- // that includes the only the initial rows put by Tx1
- tm.rollback(tx2);
-
- Transaction txScan = tm.begin();
- rs = txTable.getScanner(txScan, new Scan().setStartRow(startRow).setStopRow(stopRow));
- r = rs.next();
- while (r != null) {
- LOG.trace("Scan1 :" + Bytes.toString(r.getRow()) + " => " + Bytes.toString(r.getValue(fam, col)));
- assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
- "Unexpected value for SI scan " + txScan + ": " + Bytes.toString(r.getValue(fam, col)));
- r = rs.next();
- }
-
- // Same check as before but checking that the results are correct when retrieved through the Scanner Iterator
- ResultScanner iterableRS = txTable.getScanner(txScan, new Scan().setStartRow(startRow).setStopRow(stopRow));
- for (Result result : iterableRS) {
- assertTrue(Bytes.equals(data1, result.getValue(fam, col)),
- "Unexpected value for SI scan " + txScan + ": " + Bytes.toString(result.getValue(fam, col)));
- }
-
- // Finally, check that the Scanner Iterator does not implement the remove method
- try {
- iterableRS.iterator().remove();
- fail();
- } catch (RuntimeException re) {
- // Expected
- }
-
- }
-
-}
[28/52] [abbrv] incubator-omid git commit: Move com.yahoo ->
org.apache
Posted by ik...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
----------------------------------------------------------------------
diff --git a/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java b/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
deleted file mode 100644
index 2577e40..0000000
--- a/hbase-tools/src/main/java/com/yahoo/omid/tools/hbase/OmidTableManager.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tools.hbase;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-import com.beust.jcommander.Parameters;
-import com.beust.jcommander.ParametersDelegate;
-import com.yahoo.omid.HBaseShims;
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.committable.hbase.KeyGenerator;
-import com.yahoo.omid.committable.hbase.KeyGeneratorImplementations;
-import com.yahoo.omid.committable.hbase.RegionSplitter;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Helper class to create required HBase tables by Omid
- */
-public class OmidTableManager {
-
- private static final Logger LOG = LoggerFactory.getLogger(OmidTableManager.class);
-
- public static final String COMMIT_TABLE_COMMAND_NAME = "commit-table";
- static final String TIMESTAMP_TABLE_COMMAND_NAME = "timestamp-table";
-
- private static final byte[][] commitTableFamilies = new byte[][]{
- HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME.getBytes(),
- HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME.getBytes()};
- private static final byte[][] timestampTableFamilies = new byte[][]{
- HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME.getBytes()};
-
- private JCommander commandLine;
- private MainConfig mainConfig = new MainConfig();
- private CommitTableCommand commitTableCommand = new CommitTableCommand();
- private TimestampTableCommand timestampTableCommand = new TimestampTableCommand();
-
- public OmidTableManager(String... args) {
- commandLine = new JCommander(mainConfig);
- commandLine.addCommand(COMMIT_TABLE_COMMAND_NAME, commitTableCommand);
- commandLine.addCommand(TIMESTAMP_TABLE_COMMAND_NAME, timestampTableCommand);
- try {
- commandLine.parse(args);
- } catch (ParameterException ex) {
- commandLine.usage();
- throw new IllegalArgumentException(ex.getMessage());
- }
- }
-
- public void executeActionsOnHBase(Configuration hbaseConf) throws IOException {
-
- HBaseLogin.loginIfNeeded(mainConfig.loginFlags);
-
- try (HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseConf)) {
- byte[][] tableFamilies;
- byte[][] splitKeys = new byte[0][0];
- String tableName;
-
- LOG.info("----------------------------------------------------------------------------------------------");
- switch (commandLine.getParsedCommand()) {
- case COMMIT_TABLE_COMMAND_NAME:
- LOG.info("Performing actions related to COMMIT TABLE");
- tableName = commitTableCommand.tableName;
- tableFamilies = commitTableFamilies;
- if (commitTableCommand.numRegions > 1) {
- splitKeys = splitInUniformRegions(hbaseConf, commitTableCommand.numRegions);
- }
- break;
- case TIMESTAMP_TABLE_COMMAND_NAME:
- LOG.info("Performing actions related to TIMESTAMP TABLE");
- tableName = timestampTableCommand.tableName;
- tableFamilies = timestampTableFamilies;
- break;
- default:
- LOG.error("Unknown command: {}", commandLine.getParsedCommand());
- commandLine.usage();
- return;
- }
-
- createTable(hBaseAdmin, tableName, tableFamilies, splitKeys, 1);
- LOG.info("----------------------------------------------------------------------------------------------");
-
- }
- }
-
- public static void main(String... args) throws Exception {
-
- OmidTableManager tableManager = new OmidTableManager(args);
- tableManager.executeActionsOnHBase(HBaseConfiguration.create());
-
- }
-
- // ----------------------------------------------------------------------------------------------------------------
- // Helper methods and classes
- // ----------------------------------------------------------------------------------------------------------------
-
- private static byte[][] splitInUniformRegions(Configuration hBaseConf, int numRegions) throws IOException {
-
- KeyGenerator keyGen = KeyGeneratorImplementations.defaultKeyGenerator();
- RegionSplitter.SplitAlgorithm algo =
- RegionSplitter.newSplitAlgoInstance(hBaseConf, RegionSplitter.UniformSplit.class.getName());
- algo.setFirstRow(algo.rowToStr(keyGen.startTimestampToKey(0)));
- algo.setLastRow(algo.rowToStr(keyGen.startTimestampToKey(Long.MAX_VALUE)));
-
- // Return the split keys
- return algo.split(numRegions);
-
- }
-
- private static void createTable(HBaseAdmin admin, String tableName, byte[][] families, byte[][] splitKeys,
- int maxVersions)
- throws IOException {
-
- LOG.info("About to create Table named {} with {} splits", tableName, splitKeys.length);
-
- if (admin.tableExists(tableName)) {
- LOG.error("Table {} already exists. Table creation cancelled", tableName);
- return;
- }
-
- HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
-
- for (byte[] family : families) {
- HColumnDescriptor colDescriptor = new HColumnDescriptor(family);
- colDescriptor.setMaxVersions(maxVersions);
- HBaseShims.addFamilyToHTableDescriptor(tableDescriptor, colDescriptor);
- LOG.info("\tAdding Family {}", colDescriptor);
- }
-
- admin.createTable(tableDescriptor, splitKeys);
-
- LOG.info("Table {} created. Regions: {}", tableName, admin.getTableRegions(Bytes.toBytes(tableName)).size());
-
- }
-
- // Configuration-related classes
-
- static class MainConfig {
-
- @ParametersDelegate
- SecureHBaseConfig loginFlags = new SecureHBaseConfig();
-
- }
-
- @Parameters(commandDescription = "Specifies configuration for the Commit Table")
- static class CommitTableCommand {
-
- @Parameter(names = "-tableName", description = "Table name where to stores the commits", required = false)
- String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
-
- @Parameter(names = "-numRegions", description = "Number of splits (to pre-split tableName)", required = false,
- validateWith = IntegerGreaterThanZero.class)
- int numRegions = 16;
-
- }
-
- @Parameters(commandDescription = "Specifies configuration for the Timestamp Table")
- static class TimestampTableCommand {
-
- @Parameter(names = "-tableName", description = "Table name where to store timestamps")
- String tableName = HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
-
- }
-
- public static class IntegerGreaterThanZero implements IParameterValidator {
-
- public void validate(String name, String value) throws ParameterException {
- int n = Integer.parseInt(value);
- if (n <= 0) {
- throw new ParameterException("Parameter " + name + " should be > 0 (found " + value + ")");
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-tools/src/main/java/org/apache/omid/tools/hbase/OmidTableManager.java
----------------------------------------------------------------------
diff --git a/hbase-tools/src/main/java/org/apache/omid/tools/hbase/OmidTableManager.java b/hbase-tools/src/main/java/org/apache/omid/tools/hbase/OmidTableManager.java
new file mode 100644
index 0000000..a6a19d8
--- /dev/null
+++ b/hbase-tools/src/main/java/org/apache/omid/tools/hbase/OmidTableManager.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tools.hbase;
+
+import com.beust.jcommander.IParameterValidator;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+import org.apache.omid.HBaseShims;
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.committable.hbase.KeyGenerator;
+import org.apache.omid.committable.hbase.KeyGeneratorImplementations;
+import org.apache.omid.committable.hbase.RegionSplitter;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Helper class to create required HBase tables by Omid
+ */
+public class OmidTableManager {
+
+ private static final Logger LOG = LoggerFactory.getLogger(OmidTableManager.class);
+
+ public static final String COMMIT_TABLE_COMMAND_NAME = "commit-table";
+ static final String TIMESTAMP_TABLE_COMMAND_NAME = "timestamp-table";
+
+ private static final byte[][] commitTableFamilies = new byte[][]{
+ HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_CF_NAME.getBytes(),
+ HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_LWM_CF_NAME.getBytes()};
+ private static final byte[][] timestampTableFamilies = new byte[][]{
+ HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_CF_NAME.getBytes()};
+
+ private JCommander commandLine;
+ private MainConfig mainConfig = new MainConfig();
+ private CommitTableCommand commitTableCommand = new CommitTableCommand();
+ private TimestampTableCommand timestampTableCommand = new TimestampTableCommand();
+
+ public OmidTableManager(String... args) {
+ commandLine = new JCommander(mainConfig);
+ commandLine.addCommand(COMMIT_TABLE_COMMAND_NAME, commitTableCommand);
+ commandLine.addCommand(TIMESTAMP_TABLE_COMMAND_NAME, timestampTableCommand);
+ try {
+ commandLine.parse(args);
+ } catch (ParameterException ex) {
+ commandLine.usage();
+ throw new IllegalArgumentException(ex.getMessage());
+ }
+ }
+
+ public void executeActionsOnHBase(Configuration hbaseConf) throws IOException {
+
+ HBaseLogin.loginIfNeeded(mainConfig.loginFlags);
+
+ try (HBaseAdmin hBaseAdmin = new HBaseAdmin(hbaseConf)) {
+ byte[][] tableFamilies;
+ byte[][] splitKeys = new byte[0][0];
+ String tableName;
+
+ LOG.info("----------------------------------------------------------------------------------------------");
+ switch (commandLine.getParsedCommand()) {
+ case COMMIT_TABLE_COMMAND_NAME:
+ LOG.info("Performing actions related to COMMIT TABLE");
+ tableName = commitTableCommand.tableName;
+ tableFamilies = commitTableFamilies;
+ if (commitTableCommand.numRegions > 1) {
+ splitKeys = splitInUniformRegions(hbaseConf, commitTableCommand.numRegions);
+ }
+ break;
+ case TIMESTAMP_TABLE_COMMAND_NAME:
+ LOG.info("Performing actions related to TIMESTAMP TABLE");
+ tableName = timestampTableCommand.tableName;
+ tableFamilies = timestampTableFamilies;
+ break;
+ default:
+ LOG.error("Unknown command: {}", commandLine.getParsedCommand());
+ commandLine.usage();
+ return;
+ }
+
+ createTable(hBaseAdmin, tableName, tableFamilies, splitKeys, 1);
+ LOG.info("----------------------------------------------------------------------------------------------");
+
+ }
+ }
+
+ public static void main(String... args) throws Exception {
+
+ OmidTableManager tableManager = new OmidTableManager(args);
+ tableManager.executeActionsOnHBase(HBaseConfiguration.create());
+
+ }
+
+ // ----------------------------------------------------------------------------------------------------------------
+ // Helper methods and classes
+ // ----------------------------------------------------------------------------------------------------------------
+
+ private static byte[][] splitInUniformRegions(Configuration hBaseConf, int numRegions) throws IOException {
+
+ KeyGenerator keyGen = KeyGeneratorImplementations.defaultKeyGenerator();
+ RegionSplitter.SplitAlgorithm algo =
+ RegionSplitter.newSplitAlgoInstance(hBaseConf, RegionSplitter.UniformSplit.class.getName());
+ algo.setFirstRow(algo.rowToStr(keyGen.startTimestampToKey(0)));
+ algo.setLastRow(algo.rowToStr(keyGen.startTimestampToKey(Long.MAX_VALUE)));
+
+ // Return the split keys
+ return algo.split(numRegions);
+
+ }
+
+ private static void createTable(HBaseAdmin admin, String tableName, byte[][] families, byte[][] splitKeys,
+ int maxVersions)
+ throws IOException {
+
+ LOG.info("About to create Table named {} with {} splits", tableName, splitKeys.length);
+
+ if (admin.tableExists(tableName)) {
+ LOG.error("Table {} already exists. Table creation cancelled", tableName);
+ return;
+ }
+
+ HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tableName));
+
+ for (byte[] family : families) {
+ HColumnDescriptor colDescriptor = new HColumnDescriptor(family);
+ colDescriptor.setMaxVersions(maxVersions);
+ HBaseShims.addFamilyToHTableDescriptor(tableDescriptor, colDescriptor);
+ LOG.info("\tAdding Family {}", colDescriptor);
+ }
+
+ admin.createTable(tableDescriptor, splitKeys);
+
+ LOG.info("Table {} created. Regions: {}", tableName, admin.getTableRegions(Bytes.toBytes(tableName)).size());
+
+ }
+
+ // Configuration-related classes
+
+ static class MainConfig {
+
+ @ParametersDelegate
+ SecureHBaseConfig loginFlags = new SecureHBaseConfig();
+
+ }
+
+ @Parameters(commandDescription = "Specifies configuration for the Commit Table")
+ static class CommitTableCommand {
+
+ @Parameter(names = "-tableName", description = "Table name where to stores the commits", required = false)
+ String tableName = HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME;
+
+ @Parameter(names = "-numRegions", description = "Number of splits (to pre-split tableName)", required = false,
+ validateWith = IntegerGreaterThanZero.class)
+ int numRegions = 16;
+
+ }
+
+ @Parameters(commandDescription = "Specifies configuration for the Timestamp Table")
+ static class TimestampTableCommand {
+
+ @Parameter(names = "-tableName", description = "Table name where to store timestamps")
+ String tableName = HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME;
+
+ }
+
+ public static class IntegerGreaterThanZero implements IParameterValidator {
+
+ public void validate(String name, String value) throws ParameterException {
+ int n = Integer.parseInt(value);
+ if (n <= 0) {
+ throw new ParameterException("Parameter " + name + " should be > 0 (found " + value + ")");
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
----------------------------------------------------------------------
diff --git a/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java b/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
deleted file mode 100644
index 7773eeb..0000000
--- a/hbase-tools/src/test/java/com/yahoo/omid/tools/hbase/TestOmidTableManager.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.tools.hbase;
-
-import com.yahoo.omid.committable.hbase.HBaseCommitTableConfig;
-import com.yahoo.omid.timestamp.storage.HBaseTimestampStorageConfig;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static com.yahoo.omid.tools.hbase.OmidTableManager.COMMIT_TABLE_COMMAND_NAME;
-import static com.yahoo.omid.tools.hbase.OmidTableManager.TIMESTAMP_TABLE_COMMAND_NAME;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-public class TestOmidTableManager {
-
- private HBaseTestingUtility hBaseTestUtil;
- private Configuration hbaseConf;
- private HBaseAdmin hBaseAdmin;
-
- @BeforeClass
- public void setUpClass() throws Exception {
- // HBase setup
- hbaseConf = HBaseConfiguration.create();
-
- hBaseTestUtil = new HBaseTestingUtility(hbaseConf);
- hBaseTestUtil.startMiniCluster(1);
-
- hBaseAdmin = hBaseTestUtil.getHBaseAdmin();
- }
-
- @AfterClass
- public void tearDownClass() throws Exception {
-
- hBaseAdmin.close();
-
- hBaseTestUtil.shutdownMiniCluster();
-
- }
-
- @Test(timeOut = 20_000)
- public void testCreateDefaultTimestampTableSucceeds() throws Throwable {
-
- String[] args = new String[]{TIMESTAMP_TABLE_COMMAND_NAME};
-
- OmidTableManager omidTableManager = new OmidTableManager(args);
- omidTableManager.executeActionsOnHBase(hbaseConf);
-
- TableName tableName = TableName.valueOf(HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
-
- assertTrue(hBaseAdmin.tableExists(tableName));
- int numRegions = hBaseAdmin.getTableRegions(tableName).size();
- assertEquals(numRegions, 1, "Should have only 1 region");
-
- }
-
- @Test(timeOut = 20_000)
- public void testCreateDefaultCommitTableSucceeds() throws Throwable {
-
- String[] args = new String[]{COMMIT_TABLE_COMMAND_NAME};
-
- OmidTableManager omidTableManager = new OmidTableManager(args);
- omidTableManager.executeActionsOnHBase(hbaseConf);
-
- TableName tableName = TableName.valueOf(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME);
-
- assertTrue(hBaseAdmin.tableExists(tableName));
- int numRegions = hBaseAdmin.getTableRegions(tableName).size();
- assertEquals(numRegions, 16, "Should have 16 regions");
-
- }
-
- @Test(timeOut = 20_000)
- public void testCreateCustomCommitTableSucceeds() throws Throwable {
-
- String[] args = new String[]{COMMIT_TABLE_COMMAND_NAME, "-tableName", "my-commit-table", "-numRegions", "1"};
-
- OmidTableManager omidTableManager = new OmidTableManager(args);
- omidTableManager.executeActionsOnHBase(hbaseConf);
-
- TableName tableName = TableName.valueOf("my-commit-table");
-
- assertTrue(hBaseAdmin.tableExists(tableName));
- int numRegions = hBaseAdmin.getTableRegions(tableName).size();
- assertEquals(numRegions, 1, "Should have only 1 regions");
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class, timeOut = 20_000)
- public void testExceptionIsThrownWhenSpecifyingAWrongCommand() throws Throwable {
-
- String[] args = new String[]{"non-recognized-command"};
-
- OmidTableManager omidTableManager = new OmidTableManager(args);
- omidTableManager.executeActionsOnHBase(hbaseConf);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/hbase-tools/src/test/java/org/apache/omid/tools/hbase/TestOmidTableManager.java
----------------------------------------------------------------------
diff --git a/hbase-tools/src/test/java/org/apache/omid/tools/hbase/TestOmidTableManager.java b/hbase-tools/src/test/java/org/apache/omid/tools/hbase/TestOmidTableManager.java
new file mode 100644
index 0000000..a67774c
--- /dev/null
+++ b/hbase-tools/src/test/java/org/apache/omid/tools/hbase/TestOmidTableManager.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.tools.hbase;
+
+import org.apache.omid.committable.hbase.HBaseCommitTableConfig;
+import org.apache.omid.timestamp.storage.HBaseTimestampStorageConfig;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.apache.omid.tools.hbase.OmidTableManager.COMMIT_TABLE_COMMAND_NAME;
+import static org.apache.omid.tools.hbase.OmidTableManager.TIMESTAMP_TABLE_COMMAND_NAME;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class TestOmidTableManager {
+
+ private HBaseTestingUtility hBaseTestUtil;
+ private Configuration hbaseConf;
+ private HBaseAdmin hBaseAdmin;
+
+ @BeforeClass
+ public void setUpClass() throws Exception {
+ // HBase setup
+ hbaseConf = HBaseConfiguration.create();
+
+ hBaseTestUtil = new HBaseTestingUtility(hbaseConf);
+ hBaseTestUtil.startMiniCluster(1);
+
+ hBaseAdmin = hBaseTestUtil.getHBaseAdmin();
+ }
+
+ @AfterClass
+ public void tearDownClass() throws Exception {
+
+ hBaseAdmin.close();
+
+ hBaseTestUtil.shutdownMiniCluster();
+
+ }
+
+ @Test(timeOut = 20_000)
+ public void testCreateDefaultTimestampTableSucceeds() throws Throwable {
+
+ String[] args = new String[]{TIMESTAMP_TABLE_COMMAND_NAME};
+
+ OmidTableManager omidTableManager = new OmidTableManager(args);
+ omidTableManager.executeActionsOnHBase(hbaseConf);
+
+ TableName tableName = TableName.valueOf(HBaseTimestampStorageConfig.DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME);
+
+ assertTrue(hBaseAdmin.tableExists(tableName));
+ int numRegions = hBaseAdmin.getTableRegions(tableName).size();
+ assertEquals(numRegions, 1, "Should have only 1 region");
+
+ }
+
+ @Test(timeOut = 20_000)
+ public void testCreateDefaultCommitTableSucceeds() throws Throwable {
+
+ String[] args = new String[]{COMMIT_TABLE_COMMAND_NAME};
+
+ OmidTableManager omidTableManager = new OmidTableManager(args);
+ omidTableManager.executeActionsOnHBase(hbaseConf);
+
+ TableName tableName = TableName.valueOf(HBaseCommitTableConfig.DEFAULT_COMMIT_TABLE_NAME);
+
+ assertTrue(hBaseAdmin.tableExists(tableName));
+ int numRegions = hBaseAdmin.getTableRegions(tableName).size();
+ assertEquals(numRegions, 16, "Should have 16 regions");
+
+ }
+
+ @Test(timeOut = 20_000)
+ public void testCreateCustomCommitTableSucceeds() throws Throwable {
+
+ String[] args = new String[]{COMMIT_TABLE_COMMAND_NAME, "-tableName", "my-commit-table", "-numRegions", "1"};
+
+ OmidTableManager omidTableManager = new OmidTableManager(args);
+ omidTableManager.executeActionsOnHBase(hbaseConf);
+
+ TableName tableName = TableName.valueOf("my-commit-table");
+
+ assertTrue(hBaseAdmin.tableExists(tableName));
+ int numRegions = hBaseAdmin.getTableRegions(tableName).size();
+ assertEquals(numRegions, 1, "Should have only 1 regions");
+ }
+
+ @Test(expectedExceptions = IllegalArgumentException.class, timeOut = 20_000)
+ public void testExceptionIsThrownWhenSpecifyingAWrongCommand() throws Throwable {
+
+ String[] args = new String[]{"non-recognized-command"};
+
+ OmidTableManager omidTableManager = new OmidTableManager(args);
+ omidTableManager.executeActionsOnHBase(hbaseConf);
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/pom.xml
----------------------------------------------------------------------
diff --git a/metrics/pom.xml b/metrics/pom.xml
index 7950f57..5e1b032 100644
--- a/metrics/pom.xml
+++ b/metrics/pom.xml
@@ -5,7 +5,7 @@
<parent>
<artifactId>omid</artifactId>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java b/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
deleted file mode 100644
index d8d340a..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/AbstractMetricsConfig.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-import com.google.inject.Inject;
-
-import javax.inject.Named;
-import javax.inject.Singleton;
-
-@Singleton
-public abstract class AbstractMetricsConfig {
-
- private static final int DEFAULT_OUTPUT_FREQ_IN_SECS = 60;
-
- private static final String OUTPUT_FREQ_IN_SECS_KEY = "metrics.output.frequency.secs";
-
- private int outputFreqInSecs = DEFAULT_OUTPUT_FREQ_IN_SECS;
-
- public int getOutputFreqInSecs() {
- return outputFreqInSecs;
- }
-
- @Inject(optional = true)
- public void setOutputFreqInSecs(@Named(OUTPUT_FREQ_IN_SECS_KEY) int outputFreqInSecs) {
- this.outputFreqInSecs = outputFreqInSecs;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java b/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
deleted file mode 100644
index 11de43c..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Counter.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-public interface Counter extends Metric {
-
- /**
- * Increment the counter by one.
- */
- void inc();
-
- /**
- * Increment the counter by {@code n}.
- *
- * @param n the amount by which the counter will be increased
- */
- void inc(long n);
-
- /**
- * Decrement the counter by one.
- */
- void dec();
-
- /**
- * Decrement the counter by {@code n}.
- *
- * @param n the amount by which the counter will be decreased
- */
- void dec(long n);
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java b/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
deleted file mode 100644
index 3084803..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Gauge.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-/**
- * A gauge returns the value of a metric measured at a specific point in time.
- * For example the current size of. The value of T must be some numeric type.
- */
-public interface Gauge<T extends Number> extends Metric {
-
- T getValue();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java b/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
deleted file mode 100644
index a8c07cb..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Histogram.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-public interface Histogram extends Metric {
-
- /**
- * Adds a recorded value.
- *
- * @param value the length of the value
- */
- void update(int value);
-
- /**
- * Adds a recorded value.
- *
- * @param value the length of the value
- */
- void update(long value);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java b/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
deleted file mode 100644
index 9a582f9..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Meter.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-public interface Meter extends Metric {
-
- /**
- * Mark the occurrence of an event.
- */
- void mark();
-
- /**
- * Mark the occurrence of a given number of events.
- *
- * @param n the number of events
- */
- void mark(long n);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java b/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
deleted file mode 100644
index 733c05f..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Metric.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-/**
- * Marker interface for distinguish metrics
- */
-public interface Metric {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
deleted file mode 100644
index 515a3da..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsProvider.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-/**
- * Provider to provide metrics logger for different scopes.
- */
-public interface MetricsProvider {
-
- String CODAHALE_METRICS_CONFIG = "console:_:60:SECONDS";
-
- enum Provider {
- CODAHALE, YMON;
- }
-
- /**
- * Intialize the metrics provider.
- */
- void startMetrics();
-
- /**
- * Close the metrics provider.
- */
- void stopMetrics();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
deleted file mode 100644
index 16c49f1..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistry.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-public interface MetricsRegistry {
-
- /**
- * Registers the {@link Gauge} under the given name.
- *
- * @param name the name of the metric
- * @returns a new {@link Counter}
- */
- <T extends Number> void gauge(String name, Gauge<T> gauge);
-
- /**
- * Creates a new {@link Counter} and registers it under the given name.
- *
- * @param name the name of the metric
- * @return a new {@link Counter}
- */
- Counter counter(String name);
-
- /**
- * Creates a new {@link Timer} and registers it under the given name.
- *
- * @param name the name of the metric
- * @return a new {@link Timer}
- */
- Timer timer(String name);
-
- /**
- * Creates a new {@link Meter} and registers it under the given name.
- *
- * @param name the name of the metric
- * @return a new {@link Meter}
- */
- Meter meter(String name);
-
- /**
- * Creates a new {@link Histogram} and registers it under the given name.
- *
- * @param name the name of the metric
- * @return a new {@link Histogram}
- */
- Histogram histogram(String name);
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
deleted file mode 100644
index a96fc74..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsRegistryMap.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-import com.google.common.base.Optional;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-public class MetricsRegistryMap {
-
- private final ConcurrentMap<String, Metric> metrics = new ConcurrentHashMap<>();
-
- interface MetricBuilder<T extends Metric> {
-
- MetricBuilder<Gauge<? extends Number>> GAUGES = new MetricBuilder<Gauge<? extends Number>>() {
- @Override
- public boolean isInstance(Metric metric) {
- return Gauge.class.isInstance(metric);
- }
- };
-
- MetricBuilder<Counter> COUNTERS = new MetricBuilder<Counter>() {
- @Override
- public boolean isInstance(Metric metric) {
- return Counter.class.isInstance(metric);
- }
- };
-
- MetricBuilder<Timer> TIMERS = new MetricBuilder<Timer>() {
- @Override
- public boolean isInstance(Metric metric) {
- return Timer.class.isInstance(metric);
- }
- };
-
- MetricBuilder<Meter> METERS = new MetricBuilder<Meter>() {
- @Override
- public boolean isInstance(Metric metric) {
- return Meter.class.isInstance(metric);
- }
- };
-
- MetricBuilder<Histogram> HISTOGRAMS = new MetricBuilder<Histogram>() {
- @Override
- public boolean isInstance(Metric metric) {
- return Histogram.class.isInstance(metric);
- }
- };
-
- boolean isInstance(Metric metric);
- }
-
- public Optional<? extends Metric> get(final String name,
- MetricBuilder<? extends Metric> builder,
- Class<? extends Metric> type) {
-
- final Metric metric = metrics.get(name);
- if (builder.isInstance(metric)) {
- return Optional.of(type.cast(metric));
- }
- return Optional.absent();
-
- }
-
- @SuppressWarnings("unchecked")
- public <T extends Metric, U extends Number> List<Gauge<U>> getGauges() {
- List<Gauge<U>> gaugesList = new ArrayList<>();
- for (Metric metric : metrics.values()) {
- if (metric instanceof Gauge) {
- gaugesList.add((Gauge<U>) metric);
- }
- }
- return gaugesList;
- }
-
- public void register(String name, Metric metric) throws IllegalArgumentException {
- final Metric existing = metrics.putIfAbsent(name, metric);
- if (existing != null) {
- throw new IllegalArgumentException("A metric named " +
- name +
- " of class " +
- metric.getClass().getCanonicalName() +
- " already exists");
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java b/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
deleted file mode 100644
index 1fb2d8c..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/MetricsUtils.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-public class MetricsUtils {
-
- private static final char DEFAULT_SEPARATOR = '.';
-
- public static String name(String name, String... otherNames) {
- return name(name, DEFAULT_SEPARATOR, otherNames);
- }
-
- public static String name(String name, char separator, String... otherNames) {
- final StringBuffer builder = new StringBuffer(name);
- if (otherNames != null) {
- for (String otherName : otherNames) {
- concat(builder, otherName, separator);
- }
- }
- return builder.toString();
- }
-
- private static void concat(StringBuffer head, String tail, char separator) {
- if (tail != null && !tail.isEmpty()) {
- if (head.length() > 0) {
- head.append(separator);
- }
- head.append(tail);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java b/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
deleted file mode 100644
index 01c906a..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/NullMetricsProvider.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class NullMetricsProvider implements MetricsProvider, MetricsRegistry {
-
- private static final Logger LOG = LoggerFactory.getLogger(NullMetricsProvider.class);
-
- public NullMetricsProvider() {
- }
-
- /* ********************************** MetricsProvider interface implementation ********************************** */
-
- @Override
- public void startMetrics() {
- LOG.info("Null metrics provider started");
- }
-
- @Override
- public void stopMetrics() {
- LOG.info("Null metrics provider stopped");
- }
-
- /* ********************************** MetricsRegistry interface implementation ********************************** */
-
- @Override
- public <T extends Number> void gauge(String name, Gauge<T> gauge) {
- }
-
- @Override
- public Counter counter(final String name) {
- return new Counter() {
-
- @Override
- public void inc() {
- // Do nothing
- }
-
- @Override
- public void inc(long n) {
- // Do nothing
- }
-
- @Override
- public void dec() {
- // Do nothing
- }
-
- @Override
- public void dec(long n) {
- // Do nothing
- }
-
- };
-
- }
-
- @Override
- public Timer timer(final String name) {
-
- return new Timer() {
- @Override
- public void start() {
- // Do nothing
- }
-
- @Override
- public void stop() {
- // Do nothing
- }
-
- @Override
- public void update(long duration) {
- // Do nothing
- }
- };
-
- }
-
- @Override
- public Meter meter(final String name) {
-
- return new Meter() {
-
- @Override
- public void mark() {
- // Do nothing
- }
-
- @Override
- public void mark(long n) {
- // Do nothing
- }
-
- };
- }
-
- @Override
- public Histogram histogram(final String name) {
-
- return new Histogram() {
-
- @Override
- public void update(long value) {
- // Do nothing
- }
-
- @Override
- public void update(int value) {
- // Do nothing
- }
- };
- }
-
- /* ********************************************** Private methods *********************************************** */
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java b/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
deleted file mode 100644
index 8a22582..0000000
--- a/metrics/src/main/java/com/yahoo/omid/metrics/Timer.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.yahoo.omid.metrics;
-
-public interface Timer extends Metric {
-
- void start();
-
- void stop();
-
- void update(long durationInNs);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/AbstractMetricsConfig.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/AbstractMetricsConfig.java b/metrics/src/main/java/org/apache/omid/metrics/AbstractMetricsConfig.java
new file mode 100644
index 0000000..7d080f9
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/AbstractMetricsConfig.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+import com.google.inject.Inject;
+
+import javax.inject.Named;
+import javax.inject.Singleton;
+
+@Singleton
+public abstract class AbstractMetricsConfig {
+
+ private static final int DEFAULT_OUTPUT_FREQ_IN_SECS = 60;
+
+ private static final String OUTPUT_FREQ_IN_SECS_KEY = "metrics.output.frequency.secs";
+
+ private int outputFreqInSecs = DEFAULT_OUTPUT_FREQ_IN_SECS;
+
+ public int getOutputFreqInSecs() {
+ return outputFreqInSecs;
+ }
+
+ @Inject(optional = true)
+ public void setOutputFreqInSecs(@Named(OUTPUT_FREQ_IN_SECS_KEY) int outputFreqInSecs) {
+ this.outputFreqInSecs = outputFreqInSecs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/Counter.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/Counter.java b/metrics/src/main/java/org/apache/omid/metrics/Counter.java
new file mode 100644
index 0000000..0340be0
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/Counter.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+public interface Counter extends Metric {
+
+ /**
+ * Increment the counter by one.
+ */
+ void inc();
+
+ /**
+ * Increment the counter by {@code n}.
+ *
+ * @param n the amount by which the counter will be increased
+ */
+ void inc(long n);
+
+ /**
+ * Decrement the counter by one.
+ */
+ void dec();
+
+ /**
+ * Decrement the counter by {@code n}.
+ *
+ * @param n the amount by which the counter will be decreased
+ */
+ void dec(long n);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/Gauge.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/Gauge.java b/metrics/src/main/java/org/apache/omid/metrics/Gauge.java
new file mode 100644
index 0000000..22bfe2c
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/Gauge.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+/**
+ * A gauge returns the value of a metric measured at a specific point in time.
+ * For example the current size of. The value of T must be some numeric type.
+ */
+public interface Gauge<T extends Number> extends Metric {
+
+ T getValue();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/Histogram.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/Histogram.java b/metrics/src/main/java/org/apache/omid/metrics/Histogram.java
new file mode 100644
index 0000000..382902f
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/Histogram.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+public interface Histogram extends Metric {
+
+ /**
+ * Adds a recorded value.
+ *
+ * @param value the length of the value
+ */
+ void update(int value);
+
+ /**
+ * Adds a recorded value.
+ *
+ * @param value the length of the value
+ */
+ void update(long value);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/Meter.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/Meter.java b/metrics/src/main/java/org/apache/omid/metrics/Meter.java
new file mode 100644
index 0000000..7cc7f6f
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/Meter.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+public interface Meter extends Metric {
+
+ /**
+ * Mark the occurrence of an event.
+ */
+ void mark();
+
+ /**
+ * Mark the occurrence of a given number of events.
+ *
+ * @param n the number of events
+ */
+ void mark(long n);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/Metric.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/Metric.java b/metrics/src/main/java/org/apache/omid/metrics/Metric.java
new file mode 100644
index 0000000..bf873a2
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/Metric.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+/**
+ * Marker interface for distinguish metrics
+ */
+public interface Metric {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/MetricsProvider.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/MetricsProvider.java b/metrics/src/main/java/org/apache/omid/metrics/MetricsProvider.java
new file mode 100644
index 0000000..f578d18
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/MetricsProvider.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+/**
+ * Provider to provide metrics logger for different scopes.
+ */
+public interface MetricsProvider {
+
+ String CODAHALE_METRICS_CONFIG = "console:_:60:SECONDS";
+
+ enum Provider {
+ CODAHALE, YMON;
+ }
+
+ /**
+ * Intialize the metrics provider.
+ */
+ void startMetrics();
+
+ /**
+ * Close the metrics provider.
+ */
+ void stopMetrics();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistry.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistry.java b/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistry.java
new file mode 100644
index 0000000..6423d22
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistry.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+public interface MetricsRegistry {
+
+ /**
+ * Registers the {@link Gauge} under the given name.
+ *
+ * @param name the name of the metric
+ * @returns a new {@link Counter}
+ */
+ <T extends Number> void gauge(String name, Gauge<T> gauge);
+
+ /**
+ * Creates a new {@link Counter} and registers it under the given name.
+ *
+ * @param name the name of the metric
+ * @return a new {@link Counter}
+ */
+ Counter counter(String name);
+
+ /**
+ * Creates a new {@link Timer} and registers it under the given name.
+ *
+ * @param name the name of the metric
+ * @return a new {@link Timer}
+ */
+ Timer timer(String name);
+
+ /**
+ * Creates a new {@link Meter} and registers it under the given name.
+ *
+ * @param name the name of the metric
+ * @return a new {@link Meter}
+ */
+ Meter meter(String name);
+
+ /**
+ * Creates a new {@link Histogram} and registers it under the given name.
+ *
+ * @param name the name of the metric
+ * @return a new {@link Histogram}
+ */
+ Histogram histogram(String name);
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistryMap.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistryMap.java b/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistryMap.java
new file mode 100644
index 0000000..b3772d6
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/MetricsRegistryMap.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+import com.google.common.base.Optional;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public class MetricsRegistryMap {
+
+ private final ConcurrentMap<String, Metric> metrics = new ConcurrentHashMap<>();
+
+ interface MetricBuilder<T extends Metric> {
+
+ MetricBuilder<Gauge<? extends Number>> GAUGES = new MetricBuilder<Gauge<? extends Number>>() {
+ @Override
+ public boolean isInstance(Metric metric) {
+ return Gauge.class.isInstance(metric);
+ }
+ };
+
+ MetricBuilder<Counter> COUNTERS = new MetricBuilder<Counter>() {
+ @Override
+ public boolean isInstance(Metric metric) {
+ return Counter.class.isInstance(metric);
+ }
+ };
+
+ MetricBuilder<Timer> TIMERS = new MetricBuilder<Timer>() {
+ @Override
+ public boolean isInstance(Metric metric) {
+ return Timer.class.isInstance(metric);
+ }
+ };
+
+ MetricBuilder<Meter> METERS = new MetricBuilder<Meter>() {
+ @Override
+ public boolean isInstance(Metric metric) {
+ return Meter.class.isInstance(metric);
+ }
+ };
+
+ MetricBuilder<Histogram> HISTOGRAMS = new MetricBuilder<Histogram>() {
+ @Override
+ public boolean isInstance(Metric metric) {
+ return Histogram.class.isInstance(metric);
+ }
+ };
+
+ boolean isInstance(Metric metric);
+ }
+
+ public Optional<? extends Metric> get(final String name,
+ MetricBuilder<? extends Metric> builder,
+ Class<? extends Metric> type) {
+
+ final Metric metric = metrics.get(name);
+ if (builder.isInstance(metric)) {
+ return Optional.of(type.cast(metric));
+ }
+ return Optional.absent();
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public <T extends Metric, U extends Number> List<Gauge<U>> getGauges() {
+ List<Gauge<U>> gaugesList = new ArrayList<>();
+ for (Metric metric : metrics.values()) {
+ if (metric instanceof Gauge) {
+ gaugesList.add((Gauge<U>) metric);
+ }
+ }
+ return gaugesList;
+ }
+
+ public void register(String name, Metric metric) throws IllegalArgumentException {
+ final Metric existing = metrics.putIfAbsent(name, metric);
+ if (existing != null) {
+ throw new IllegalArgumentException("A metric named " +
+ name +
+ " of class " +
+ metric.getClass().getCanonicalName() +
+ " already exists");
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/MetricsUtils.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/MetricsUtils.java b/metrics/src/main/java/org/apache/omid/metrics/MetricsUtils.java
new file mode 100644
index 0000000..f9d9428
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/MetricsUtils.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+public class MetricsUtils {
+
+ private static final char DEFAULT_SEPARATOR = '.';
+
+ public static String name(String name, String... otherNames) {
+ return name(name, DEFAULT_SEPARATOR, otherNames);
+ }
+
+ public static String name(String name, char separator, String... otherNames) {
+ final StringBuffer builder = new StringBuffer(name);
+ if (otherNames != null) {
+ for (String otherName : otherNames) {
+ concat(builder, otherName, separator);
+ }
+ }
+ return builder.toString();
+ }
+
+ private static void concat(StringBuffer head, String tail, char separator) {
+ if (tail != null && !tail.isEmpty()) {
+ if (head.length() > 0) {
+ head.append(separator);
+ }
+ head.append(tail);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/NullMetricsProvider.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/NullMetricsProvider.java b/metrics/src/main/java/org/apache/omid/metrics/NullMetricsProvider.java
new file mode 100644
index 0000000..49142a2
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/NullMetricsProvider.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class NullMetricsProvider implements MetricsProvider, MetricsRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NullMetricsProvider.class);
+
+ public NullMetricsProvider() {
+ }
+
+ /* ********************************** MetricsProvider interface implementation ********************************** */
+
+ @Override
+ public void startMetrics() {
+ LOG.info("Null metrics provider started");
+ }
+
+ @Override
+ public void stopMetrics() {
+ LOG.info("Null metrics provider stopped");
+ }
+
+ /* ********************************** MetricsRegistry interface implementation ********************************** */
+
+ @Override
+ public <T extends Number> void gauge(String name, Gauge<T> gauge) {
+ }
+
+ @Override
+ public Counter counter(final String name) {
+ return new Counter() {
+
+ @Override
+ public void inc() {
+ // Do nothing
+ }
+
+ @Override
+ public void inc(long n) {
+ // Do nothing
+ }
+
+ @Override
+ public void dec() {
+ // Do nothing
+ }
+
+ @Override
+ public void dec(long n) {
+ // Do nothing
+ }
+
+ };
+
+ }
+
+ @Override
+ public Timer timer(final String name) {
+
+ return new Timer() {
+ @Override
+ public void start() {
+ // Do nothing
+ }
+
+ @Override
+ public void stop() {
+ // Do nothing
+ }
+
+ @Override
+ public void update(long duration) {
+ // Do nothing
+ }
+ };
+
+ }
+
+ @Override
+ public Meter meter(final String name) {
+
+ return new Meter() {
+
+ @Override
+ public void mark() {
+ // Do nothing
+ }
+
+ @Override
+ public void mark(long n) {
+ // Do nothing
+ }
+
+ };
+ }
+
+ @Override
+ public Histogram histogram(final String name) {
+
+ return new Histogram() {
+
+ @Override
+ public void update(long value) {
+ // Do nothing
+ }
+
+ @Override
+ public void update(int value) {
+ // Do nothing
+ }
+ };
+ }
+
+ /* ********************************************** Private methods *********************************************** */
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/metrics/src/main/java/org/apache/omid/metrics/Timer.java
----------------------------------------------------------------------
diff --git a/metrics/src/main/java/org/apache/omid/metrics/Timer.java b/metrics/src/main/java/org/apache/omid/metrics/Timer.java
new file mode 100644
index 0000000..5d94784
--- /dev/null
+++ b/metrics/src/main/java/org/apache/omid/metrics/Timer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid.metrics;
+
+public interface Timer extends Metric {
+
+ void start();
+
+ void stop();
+
+ void update(long durationInNs);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 060d35f..27d1268 100644
--- a/pom.xml
+++ b/pom.xml
@@ -8,8 +8,8 @@
<name>Omid</name>
<description>The Omid project provides transactional support for HBase-based applications</description>
<inceptionYear>2011</inceptionYear>
- <url>http://www.github.com/yahoo/omid</url>
- <groupId>com.yahoo.omid</groupId>
+ <url>http://www.github.org/apache/omid</url>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<packaging>pom</packaging>
<!-- WARNING: do not update version manually, use mvn versions:set -->
@@ -43,9 +43,9 @@
</modules>
<scm>
- <connection>scm:git:git://github.com/yahoo/omid.git</connection>
- <developerConnection>scm:git:https://${GH_TOKEN}@github.com/yahoo/omid.git</developerConnection>
- <url>https://github.com/yahoo/omid</url>
+ <connection>scm:git:git://github.org/apache/omid.git</connection>
+ <developerConnection>scm:git:https://${GH_TOKEN}@github.org/apache/omid.git</developerConnection>
+ <url>https://github.org/apache/omid</url>
<tag>omid-0.8.1.21</tag>
</scm>
@@ -221,7 +221,7 @@
</formats>
<instrumentation>
<excludes>
- <exclude>com/yahoo/omid/proto/*.class</exclude>
+ <exclude>org/apache/omid/proto/*.class</exclude>
</excludes>
</instrumentation>
</configuration>
@@ -308,16 +308,17 @@
<!-- Project specific exclusions (e.g. they've been copied/modified from other open-source projects -->
<!-- Protocol buffers generated classes -->
- <exclude>**/src/main/java/com/yahoo/omid/proto/TSOProto.java</exclude>
+ <exclude>**/src/main/java/org/apache/omid/proto/TSOProto.java</exclude>
<!-- Taken from https://github.com/brianfrankcooper/YCSB -->
- <exclude>**/src/main/java/com/yahoo/omid/benchmarks/utils/Generator.java</exclude>
- <exclude>**/src/main/java/com/yahoo/omid/benchmarks/utils/IntegerGenerator.java</exclude>
- <exclude>**/src/main/java/com/yahoo/omid/benchmarks/utils/ZipfianGenerator.java</exclude>
- <exclude>**/src/main/java/com/yahoo/omid/benchmarks/utils/ScrambledZipfianGenerator.java</exclude>
+ <exclude>**/src/main/java/org/apache/omid/benchmarks/utils/Generator.java</exclude>
+ <exclude>**/src/main/java/org/apache/omid/benchmarks/utils/IntegerGenerator.java</exclude>
+ <exclude>**/src/main/java/org/apache/omid/benchmarks/utils/ZipfianGenerator.java</exclude>
+ <exclude>**/src/main/java/org/apache/omid/benchmarks/utils/ScrambledZipfianGenerator.java
+ </exclude>
<!-- Taken from https://github.com/apache/hbase -->
- <exclude>**/src/main/java/com/yahoo/omid/committable/hbase/RegionSplitter.java</exclude>
+ <exclude>**/src/main/java/org/apache/omid/committable/hbase/RegionSplitter.java</exclude>
</excludes>
<!-- Add comment style for protocol buffers -->
http://git-wip-us.apache.org/repos/asf/incubator-omid/blob/9cd856c6/statemachine/pom.xml
----------------------------------------------------------------------
diff --git a/statemachine/pom.xml b/statemachine/pom.xml
index eff5fb2..660770e 100644
--- a/statemachine/pom.xml
+++ b/statemachine/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>com.yahoo.omid</groupId>
+ <groupId>org.apache.omid</groupId>
<artifactId>omid</artifactId>
<version>0.8.1.37-SNAPSHOT</version>
</parent>
[12/52] [abbrv] incubator-omid git commit: Merge pull request #88
from yahoo/rename-tsoclient-pkg
Posted by ik...@apache.org.
Merge pull request #88 from yahoo/rename-tsoclient-pkg
Rename tsoclient package to tso.client
Project: http://git-wip-us.apache.org/repos/asf/incubator-omid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-omid/commit/b0d70342
Tree: http://git-wip-us.apache.org/repos/asf/incubator-omid/tree/b0d70342
Diff: http://git-wip-us.apache.org/repos/asf/incubator-omid/diff/b0d70342
Branch: refs/heads/master
Commit: b0d70342c8f4a1d3f648c987c579ce7a2e85db54
Parents: 516e517 c6410f7
Author: ikatkov <ik...@gmail.com>
Authored: Fri Apr 15 11:54:35 2016 -0700
Committer: ikatkov <ik...@gmail.com>
Committed: Fri Apr 15 11:54:35 2016 -0700
----------------------------------------------------------------------
.../yahoo/omid/benchmarks/tso/RawTxRunner.java | 10 +-
.../tso/TSOServerBenchmarkConfig.java | 2 +-
.../default-tso-server-benchmark-config.yml | 4 +-
.../omid/examples/ConfigurationExample.java | 2 +-
.../main/resources/hbase-omid-client-config.yml | 2 +-
.../transaction/HBaseAsyncPostCommitter.java | 2 +-
.../com/yahoo/omid/transaction/HBaseCellId.java | 2 +-
.../HBaseOmidClientConfiguration.java | 4 +-
.../transaction/HBaseSyncPostCommitter.java | 2 +-
.../transaction/HBaseTransactionManager.java | 4 +-
.../default-hbase-omid-client-config.yml | 2 +-
.../yahoo/omid/transaction/OmidTestBase.java | 4 +-
.../TestEndToEndScenariosWithHA.java | 2 +-
.../TestHBaseTransactionManager.java | 2 +-
.../yahoo/omid/transaction/TestShadowCells.java | 2 -
.../transaction/TestTransactionCleanup.java | 6 +-
.../omid/transaction/TestTxMgrFailover.java | 2 +-
.../resources/test-hbase-omid-client-config.yml | 4 +-
.../omid/transaction/AbstractTransaction.java | 2 +-
.../transaction/AbstractTransactionManager.java | 12 +-
.../omid/transaction/PostCommitActions.java | 2 +-
.../yahoo/omid/tso/client/AbortException.java | 27 +
.../java/com/yahoo/omid/tso/client/CellId.java | 24 +
.../yahoo/omid/tso/client/ClosingException.java | 28 +
.../omid/tso/client/ConnectionException.java | 30 +
.../omid/tso/client/ForwardingTSOFuture.java | 64 ++
.../tso/client/HandshakeFailedException.java | 28 +
.../yahoo/omid/tso/client/MockTSOClient.java | 102 ++
.../yahoo/omid/tso/client/NewTSOException.java | 27 +
.../tso/client/OmidClientConfiguration.java | 177 ++++
.../tso/client/ServiceUnavailableException.java | 32 +
.../com/yahoo/omid/tso/client/TSOClient.java | 933 +++++++++++++++++++
.../com/yahoo/omid/tso/client/TSOFuture.java | 25 +
.../com/yahoo/omid/tso/client/TSOProtocol.java | 56 ++
.../yahoo/omid/tso/util/DummyCellIdImpl.java | 2 +-
.../yahoo/omid/tsoclient/AbortException.java | 27 -
.../java/com/yahoo/omid/tsoclient/CellId.java | 24 -
.../yahoo/omid/tsoclient/ClosingException.java | 28 -
.../omid/tsoclient/ConnectionException.java | 30 -
.../omid/tsoclient/ForwardingTSOFuture.java | 64 --
.../tsoclient/HandshakeFailedException.java | 28 -
.../com/yahoo/omid/tsoclient/MockTSOClient.java | 102 --
.../yahoo/omid/tsoclient/NewTSOException.java | 27 -
.../omid/tsoclient/OmidClientConfiguration.java | 177 ----
.../tsoclient/ServiceUnavailableException.java | 32 -
.../com/yahoo/omid/tsoclient/TSOClient.java | 933 -------------------
.../com/yahoo/omid/tsoclient/TSOFuture.java | 25 -
.../com/yahoo/omid/tsoclient/TSOProtocol.java | 56 --
.../src/main/resources/omid-client-config.yml | 6 +-
.../omid/tso/client/TestMockTSOClient.java | 72 ++
.../tso/client/TestOmidClientConfiguration.java | 32 +
.../yahoo/omid/tsoclient/TestMockTSOClient.java | 72 --
.../tsoclient/TestOmidClientConfiguration.java | 32 -
.../com/yahoo/omid/tso/TestLeaseManager.java | 2 +-
.../omid/tso/client/TSOClientAccessor.java | 29 +
.../yahoo/omid/tso/client/TSOClientOneShot.java | 63 ++
.../com/yahoo/omid/tso/client/TSOClientRaw.java | 150 +++
...tionOfTSOClientServerBasicFunctionality.java | 233 +++++
.../client/TestTSOClientConnectionToTSO.java | 284 ++++++
...stTSOClientRequestAndResponseBehaviours.java | 423 +++++++++
.../client/TestTSOClientResponseHandling.java | 111 +++
.../tso/client/TestUnconnectedTSOClient.java | 83 ++
.../yahoo/omid/tsoclient/TSOClientAccessor.java | 30 -
.../yahoo/omid/tsoclient/TSOClientOneShot.java | 63 --
.../com/yahoo/omid/tsoclient/TSOClientRaw.java | 150 ---
...tionOfTSOClientServerBasicFunctionality.java | 233 -----
.../tsoclient/TestTSOClientConnectionToTSO.java | 285 ------
...stTSOClientRequestAndResponseBehaviours.java | 423 ---------
.../TestTSOClientResponseHandling.java | 111 ---
.../tsoclient/TestUnconnectedTSOClient.java | 84 --
70 files changed, 3074 insertions(+), 3079 deletions(-)
----------------------------------------------------------------------