You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by dj...@apache.org on 2018/10/08 09:02:36 UTC
[01/11] gora git commit: create template gora-ignite module
Repository: gora
Updated Branches:
refs/heads/master a8398651d -> 446f42ea3
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java b/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
new file mode 100644
index 0000000..1e03717
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite;
+
+import org.apache.ignite.minicluster.MiniIgniteCluster;
+import org.apache.ignite.minicluster.MiniIgniteConfig;
+import org.apache.gora.GoraTestDriver;
+import org.apache.gora.ignite.store.IgniteStore;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * @author lmcgibbn
+ *
+ */
+public class GoraIgniteTestDriver extends GoraTestDriver {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GoraIgniteTestDriver.class);
+ private static MiniIgniteCluster cluster = null;
+ private static final String PASSWORD = "drowssap";
+
+ @Rule
+ public TemporaryFolder tmpDir = new TemporaryFolder();
+
+ public GoraIgniteTestDriver() throws Exception {
+ super(IgniteStore.class);
+ }
+
+ @Override
+ public void setUpClass() throws IOException, InterruptedException {
+ log.info("Starting Ignite MiniIgniteCluster...");
+ try {
+ tmpDir.create();
+ MiniIgniteConfig miniCfg = new MiniIgniteConfig(tmpDir.getRoot(), PASSWORD);
+ miniCfg.setInstanceName("goraTest");
+ miniCfg.setZooKeeperPort(56321);
+ cluster = new MiniIgniteCluster(miniCfg);
+ cluster.start();
+ } catch (Exception e) {
+ LOG.error("Error starting Ignite MiniIgniteCluster: {}", e.getMessage());
+ // cleanup
+ tearDownClass();
+ }
+ }
+
+ @Override
+ public void tearDownClass() throws IOException, InterruptedException {
+ log.info("Shutting down Ignite MiniIgniteCluster...");
+ if (cluster != null) {
+ cluster.stop();
+ }
+ tmpDir.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
new file mode 100644
index 0000000..f536961
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Tests for <code>gora-ignite</code> including
+ * the test driver for {@link org.apache.gora.ignite.store.IgniteStoreTest}
+ */
+package org.apache.gora.ignite;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java
new file mode 100644
index 0000000..bb77bee
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import java.util.List;
+import java.util.Properties;
+import org.apache.ignite.minicluster.MiniIgniteCluster;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests authentication token is serialized correctly.
+ */
+public class AuthenticationTokenTest {
+ private static final Logger LOG = LoggerFactory.getLogger(AuthenticationTokenTest.class);
+
+ private static final String GORA_DATASTORE =
+ DataStoreFactory.GORA + "." + DataStoreFactory.DATASTORE + ".";
+ private static final String PASSWORD = "password";
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private MiniIgniteCluster cluster;
+ private DataStore<String, Employee> employeeStore;
+
+ @Before
+ @SuppressWarnings("unchecked")
+ public void setUp() throws Exception {
+ cluster = new MiniIgniteCluster(temporaryFolder.getRoot(), PASSWORD);
+ cluster.start();
+
+ Properties properties = DataStoreFactory.createProps();
+ properties.setProperty(
+ GORA_DATASTORE + IgniteStore.MOCK_PROPERTY,
+ "false");
+ properties.setProperty(
+ GORA_DATASTORE + IgniteStore.INSTANCE_NAME_PROPERTY,
+ cluster.getInstanceName());
+ properties.setProperty(
+ GORA_DATASTORE + IgniteStore.ZOOKEEPERS_NAME_PROPERTY,
+ cluster.getZooKeepers());
+ properties.setProperty(
+ GORA_DATASTORE + IgniteStore.PASSWORD_PROPERTY,
+ PASSWORD);
+
+ employeeStore = DataStoreFactory.createDataStore(
+ IgniteStore.class,
+ String.class,
+ Employee.class,
+ new Configuration(),
+ properties);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ cluster.stop();
+ }
+
+ @Test
+ public void testAuthenticationTokenIsSerializedCorrectly() throws Exception {
+ List<PartitionQuery<String, Employee>> partitions =
+ employeeStore.getPartitions(employeeStore.newQuery());
+ LOG.debug("partitions {}", partitions);
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java
new file mode 100644
index 0000000..b42d344
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import java.io.IOException;
+
+import org.apache.gora.ignite.GoraIgniteTestDriver;
+import org.apache.gora.examples.generated.Employee;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.DataStoreTestBase;
+import static org.apache.gora.store.DataStoreTestBase.log;
+import org.apache.gora.store.DataStoreTestUtil;
+import static org.apache.gora.store.DataStoreTestUtil.testResultSize;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Tests extending {@link org.apache.gora.store.DataStoreTestBase}
+ * which run the base JUnit test suite for Gora.
+ */
+public class IgniteStoreTest extends DataStoreTestBase {
+
+ static {
+ try {
+ setTestDriver(new GoraIgniteTestDriver());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ public GoraIgniteTestDriver getTestDriver() {
+ return (GoraIgniteTestDriver) testDriver;
+ }
+
+ //Until GORA-66 is resolved this test will always fail, so
+ //do not run it
+ @Ignore("skipped until GORA-66 is resolved")
+ @Override
+ public void testDeleteByQueryFields() throws IOException {
+ }
+
+ @Test
+ @Ignore("Ignite does not support Result#size() without limit set")
+ @Override
+ public void testResultSize() throws Exception {
+ }
+
+ @Test
+ @Ignore("Ignite does not support Result#size() without limit set")
+ @Override
+ public void testResultSizeStartKey() throws Exception {
+ }
+
+ @Ignore("Ignite does not support Result#size() without limit set")
+ @Override
+ public void testResultSizeEndKey() throws Exception {
+ }
+
+ @Test
+ @Ignore("Ignite does not support Result#size() without limit set")
+ @Override
+ public void testResultSizeKeyRange() throws Exception {
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java
new file mode 100644
index 0000000..1c77656
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.gora.ignite.encoders.Encoder;
+import org.apache.gora.ignite.encoders.SignedBinaryEncoder;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ */
+public class PartitionTest {
+ // TODO test more types
+
+ private static Encoder encoder = new SignedBinaryEncoder();
+
+ static long encl(long l) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ try {
+ dos.writeLong(l);
+ dos.flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return encoder.decodeLong(baos.toByteArray());
+ }
+
+ @Test
+ public void test1() throws IOException {
+ assertEquals(encl(0x006f000000000000l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {0x00, 0x6f}));
+ assertEquals(encl(1l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {0, 0, 0, 0, 0, 0, 0, 0}));
+ assertEquals(encl(0x106f000000000001l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {0x10, 0x6f, 0, 0, 0, 0, 0, 0}));
+ assertEquals(
+ encl(-1l),
+ (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
+ (byte) 0xff,
+ (byte) 0xfe}));
+
+ assertEquals(encl(0x8000000000000001l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {(byte) 0x80, 0, 0, 0, 0, 0, 0, 0}));
+ assertEquals(
+ encl(0x8000000000000000l),
+ (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
+ (byte) 0xff,
+ (byte) 0xff}));
+
+
+ try {
+ IgniteStore.followingKey(encoder, Long.class,
+ new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff});
+ assertTrue(false);
+ } catch (IllegalArgumentException iea) {
+
+ }
+ }
+
+ @Test
+ public void test2() throws IOException {
+ assertEquals(encl(0x00ffffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {0x01}));
+ assertEquals(encl(0x006effffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {0x00, 0x6f}));
+ assertEquals(encl(0xff6effffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0xff, 0x6f}));
+ assertEquals(encl(0xfffeffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0xff, (byte) 0xff}));
+ assertEquals(encl(0l), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0, 0, 0, 0, 0, 0, 0, 0}));
+
+ assertEquals(encl(0x7effffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0x7f}));
+ assertEquals(encl(0x7fffffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0x80}));
+ assertEquals(encl(0x80ffffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0x81}));
+
+ try {
+ IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0, 0, 0, 0, 0, 0, 0});
+ assertTrue(false);
+ } catch (IllegalArgumentException iea) {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
new file mode 100644
index 0000000..a41e085
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains all the unit tests for basic CRUD operations
+ * functionality of the Ignite dataStore.
+ */
+package org.apache.gora.ignite.store;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java
new file mode 100644
index 0000000..0d4e5e6
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.util;
+
+import java.nio.charset.Charset;
+import org.apache.gora.ignite.encoders.HexEncoder;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class HexEncoderTest {
+
+ @Test
+ public void testByte() {
+ HexEncoder encoder = new HexEncoder();
+
+ assertEquals("12", new String(encoder.encodeByte((byte) 0x12), Charset.defaultCharset()));
+ assertEquals("f2", new String(encoder.encodeByte((byte) 0xf2), Charset.defaultCharset()));
+
+ byte b = Byte.MIN_VALUE;
+ while (b != Byte.MAX_VALUE) {
+ assertEquals(b, encoder.decodeByte(encoder.encodeByte(b)));
+ b++;
+ }
+ }
+
+ @Test
+ public void testShort() {
+ HexEncoder encoder = new HexEncoder();
+
+ assertEquals("1234", new String(encoder.encodeShort((short) 0x1234), Charset.defaultCharset()));
+ assertEquals("f234", new String(encoder.encodeShort((short) 0xf234), Charset.defaultCharset()));
+
+ short s = Short.MIN_VALUE;
+ while (s != Short.MAX_VALUE) {
+ assertEquals(s, encoder.decodeShort(encoder.encodeShort(s)));
+ s++;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java
new file mode 100644
index 0000000..c943e73
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import org.apache.gora.ignite.encoders.SignedBinaryEncoder;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ */
+public class SignedBinaryEncoderTest {
+ @Test
+ public void testShort() throws IOException {
+ short s = Short.MIN_VALUE;
+ Text prev = null;
+
+ SignedBinaryEncoder encoder = new SignedBinaryEncoder();
+
+ while (true) {
+ byte[] enc = encoder.encodeShort(s);
+ assertEquals(s, encoder.decodeShort(enc));
+ Text current = new Text(enc);
+ if (prev != null)
+ assertTrue(prev.compareTo(current) < 0);
+ prev = current;
+ s++;
+ if (s == Short.MAX_VALUE)
+ break;
+ }
+ }
+
+ private void testInt(int start, int finish) throws IOException {
+ int i = start;
+ Text prev = null;
+
+ SignedBinaryEncoder encoder = new SignedBinaryEncoder();
+
+ while (true) {
+ byte[] enc = encoder.encodeInt(i);
+ assertEquals(i, encoder.decodeInt(enc));
+ Text current = new Text(enc);
+ if (prev != null)
+ assertTrue(prev.compareTo(current) < 0);
+ prev = current;
+ i++;
+ if (i == finish)
+ break;
+ }
+ }
+
+ @Test
+ public void testInt() throws IOException {
+ testInt(Integer.MIN_VALUE, Integer.MIN_VALUE + (1 << 16));
+ testInt(-(1 << 15), (1 << 15));
+ testInt(Integer.MAX_VALUE - (1 << 16), Integer.MAX_VALUE);
+ }
+
+ private void testLong(long start, long finish) throws IOException {
+ long l = start;
+ Text prev = null;
+
+ SignedBinaryEncoder encoder = new SignedBinaryEncoder();
+
+ while (true) {
+ byte[] enc = encoder.encodeLong(l);
+ assertEquals(l, encoder.decodeLong(enc));
+ Text current = new Text(enc);
+ if (prev != null)
+ assertTrue(prev.compareTo(current) < 0);
+ prev = current;
+ l++;
+ if (l == finish)
+ break;
+ }
+ }
+
+ @Test
+ public void testLong() throws IOException {
+ testLong(Long.MIN_VALUE, Long.MIN_VALUE + (1 << 16));
+ testLong(-(1 << 15), (1 << 15));
+ testLong(Long.MAX_VALUE - (1 << 16), Long.MAX_VALUE);
+ }
+
+ @Test
+ public void testDouble() throws IOException {
+
+ ArrayList<Double> testData = new ArrayList<>();
+ testData.add(Double.NEGATIVE_INFINITY);
+ testData.add(Double.MIN_VALUE);
+ testData.add(Math.nextUp(Double.NEGATIVE_INFINITY));
+ testData.add(Math.pow(10.0, 30.0) * -1.0);
+ testData.add(Math.pow(10.0, 30.0));
+ testData.add(Math.pow(10.0, -30.0) * -1.0);
+ testData.add(Math.pow(10.0, -30.0));
+ testData.add(Math.nextAfter(0.0, Double.NEGATIVE_INFINITY));
+ testData.add(0.0);
+ testData.add(Math.nextAfter(Double.MAX_VALUE, Double.NEGATIVE_INFINITY));
+ testData.add(Double.MAX_VALUE);
+ testData.add(Double.POSITIVE_INFINITY);
+
+ Collections.sort(testData);
+
+ SignedBinaryEncoder encoder = new SignedBinaryEncoder();
+
+ for (int i = 0; i < testData.size(); i++) {
+ byte[] enc = encoder.encodeDouble(testData.get(i));
+ assertEquals(testData.get(i), (Double) encoder.decodeDouble(enc));
+ if (i > 1) {
+ assertTrue("Checking " + testData.get(i) + " > " + testData.get(i - 1),
+ new Text(enc).compareTo(new Text(encoder.encodeDouble(testData.get(i - 1)))) > 0);
+ }
+ }
+ }
+
+ @Test
+ public void testFloat() throws IOException {
+
+ ArrayList<Float> testData = new ArrayList<>();
+ testData.add(Float.NEGATIVE_INFINITY);
+ testData.add(Float.MIN_VALUE);
+ testData.add(Math.nextUp(Float.NEGATIVE_INFINITY));
+ testData.add((float) Math.pow(10.0f, 30.0f) * -1.0f);
+ testData.add((float) Math.pow(10.0f, 30.0f));
+ testData.add((float) Math.pow(10.0f, -30.0f) * -1.0f);
+ testData.add((float) Math.pow(10.0f, -30.0f));
+ testData.add(Math.nextAfter(0.0f, Float.NEGATIVE_INFINITY));
+ testData.add(0.0f);
+ testData.add(Math.nextAfter(Float.MAX_VALUE, Float.NEGATIVE_INFINITY));
+ testData.add(Float.MAX_VALUE);
+ testData.add(Float.POSITIVE_INFINITY);
+
+ Collections.sort(testData);
+
+ SignedBinaryEncoder encoder = new SignedBinaryEncoder();
+
+ for (int i = 0; i < testData.size(); i++) {
+ byte[] enc = encoder.encodeFloat(testData.get(i));
+ assertEquals(testData.get(i), (Float)encoder.decodeFloat(enc));
+ if (i > 1) {
+ assertTrue("Checking " + testData.get(i) + " > " + testData.get(i - 1),
+ new Text(enc).compareTo(new Text(encoder.encodeFloat(testData.get(i - 1)))) > 0);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java
new file mode 100644
index 0000000..afcd619
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains all the unit tests for utils of the Ignite dataStore.
+ */
+package org.apache.gora.ignite.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/resources/gora-accumulo-mapping.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora-accumulo-mapping.xml b/gora-ignite/src/test/resources/gora-accumulo-mapping.xml
new file mode 100644
index 0000000..4d036c6
--- /dev/null
+++ b/gora-ignite/src/test/resources/gora-accumulo-mapping.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<gora-otd>
+ <table name="AccessLog">
+ <config key="table.file.compress.blocksize" value="32K"/>
+ </table>
+
+ <class name="org.apache.gora.tutorial.log.generated.Pageview" keyClass="java.lang.Long" table="AccessLog">
+ <field name="url" family="common" qualifier="url"/>
+ <field name="timestamp" family="common" qualifier="timestamp"/>
+ <field name="ip" family="common" qualifier="ip" />
+ <field name="httpMethod" family="http" qualifier="httpMethod"/>
+ <field name="httpStatusCode" family="http" qualifier="httpStatusCode"/>
+ <field name="responseSize" family="http" qualifier="responseSize"/>
+ <field name="referrer" family="misc" qualifier="referrer"/>
+ <field name="userAgent" family="misc" qualifier="userAgent"/>
+ </class>
+
+ <class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee">
+ <field name="name" family="info" qualifier="nm"/>
+ <field name="dateOfBirth" family="info" qualifier="db"/>
+ <field name="ssn" family="info" qualifier="sn"/>
+ <field name="salary" family="info" qualifier="sl"/>
+ <field name="boss" family="info" qualifier="bs"/>
+ <field name="webpage" family="info" qualifier="wp"/>
+ </class>
+
+ <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage">
+ <field name="url" family="common" qualifier="u"/>
+ <field name="content" family="content" qualifier="c"/>
+ <field name="parsedContent" family="parsedContent"/>
+ <field name="outlinks" family="outlinks"/>
+ <field name="headers" family="headers"/>
+ <field name="metadata" family="common" qualifier="metadata"/>
+ <field name="byteData" family="byteData" qualifier="byteData"/>
+ <field name="stringData" family="stringData" qualifier="stringData"/>
+ </class>
+
+ <class name="org.apache.gora.examples.generated.TokenDatum" keyClass="java.lang.String">
+ <field name="count" family="common" qualifier="count"/>
+ </class>
+</gora-otd>
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/test/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora.properties b/gora-ignite/src/test/resources/gora.properties
new file mode 100644
index 0000000..5cf0e25
--- /dev/null
+++ b/gora-ignite/src/test/resources/gora.properties
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+gora.datastore.default=org.apache.gora.ignite.store.IgniteStore
+gora.datastore.ignite.mock=false
+gora.datastore.ignite.instance=goraTest
+gora.datastore.ignite.zookeepers=localhost:56321
+gora.datastore.ignite.user=root
+gora.datastore.ignite.password=drowssap
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 48ff3b4..88b9b52 100644
--- a/pom.xml
+++ b/pom.xml
@@ -760,6 +760,7 @@
<module>gora-tutorial</module>
<module>sources-dist</module>
<module>gora-aerospike</module>
+ <module>gora-ignite</module>
</modules>
<properties>
[04/11] gora git commit: Add mapping reading and initialization for
Ignite backend
Posted by dj...@apache.org.
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
index 2f5faf9..d52d10e 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -16,1005 +16,165 @@
*/
package org.apache.gora.ignite.store;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.net.InetAddress;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.MessageFormat;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
+import java.util.Locale;
import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-import org.apache.ignite.core.client.IgniteException;
-import org.apache.ignite.core.client.IgniteSecurityException;
-import org.apache.ignite.core.client.BatchWriter;
-import org.apache.ignite.core.client.BatchWriterConfig;
-import org.apache.ignite.core.client.Connector;
-import org.apache.ignite.core.client.IsolatedScanner;
-import org.apache.ignite.core.client.IteratorSetting;
-import org.apache.ignite.core.client.MutationsRejectedException;
-import org.apache.ignite.core.client.RowIterator;
-import org.apache.ignite.core.client.Scanner;
-import org.apache.ignite.core.client.TableDeletedException;
-import org.apache.ignite.core.client.TableExistsException;
-import org.apache.ignite.core.client.TableNotFoundException;
-import org.apache.ignite.core.client.TableOfflineException;
-import org.apache.ignite.core.client.ZooKeeperInstance;
-import org.apache.ignite.core.client.impl.ClientContext;
-import org.apache.ignite.core.client.impl.Tables;
-import org.apache.ignite.core.client.impl.TabletLocator;
-import org.apache.ignite.core.client.mock.MockConnector;
-import org.apache.ignite.core.client.mock.MockInstance;
-import org.apache.ignite.core.client.mock.impl.MockTabletLocator;
-import org.apache.ignite.core.client.security.tokens.AuthenticationToken;
-import org.apache.ignite.core.client.security.tokens.PasswordToken;
-import org.apache.ignite.core.conf.IgniteConfiguration;
-import org.apache.ignite.core.data.ByteSequence;
-import org.apache.ignite.core.data.Key;
-import org.apache.ignite.core.data.impl.KeyExtent;
-import org.apache.ignite.core.data.Mutation;
-import org.apache.ignite.core.data.Range;
-import org.apache.ignite.core.data.Value;
-import org.apache.ignite.core.iterators.SortedKeyIterator;
-import org.apache.ignite.core.iterators.user.TimestampFilter;
-import org.apache.ignite.core.master.state.tables.TableState;
-import org.apache.ignite.core.security.Authorizations;
-import org.apache.ignite.core.security.ColumnVisibility;
-import org.apache.ignite.core.client.impl.Credentials;
-import org.apache.ignite.core.util.Pair;
-import org.apache.ignite.core.util.UtilWaitThread;
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.Schema.Type;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.io.BinaryDecoder;
-import org.apache.avro.io.Decoder;
-import org.apache.avro.io.DecoderFactory;
-import org.apache.avro.io.EncoderFactory;
-import org.apache.avro.specific.SpecificDatumReader;
-import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.util.Utf8;
-import org.apache.gora.ignite.encoders.BinaryEncoder;
-import org.apache.gora.ignite.encoders.Encoder;
-import org.apache.gora.ignite.query.IgniteQuery;
-import org.apache.gora.ignite.query.IgniteResult;
-import org.apache.gora.persistency.impl.DirtyListWrapper;
-import org.apache.gora.persistency.impl.DirtyMapWrapper;
+import java.util.logging.Level;
import org.apache.gora.persistency.impl.PersistentBase;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
-import org.apache.gora.query.impl.PartitionQueryImpl;
-import org.apache.gora.store.DataStoreFactory;
import org.apache.gora.store.impl.DataStoreBase;
-import org.apache.gora.util.AvroUtils;
import org.apache.gora.util.GoraException;
-import org.apache.gora.util.IOUtils;
-import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.NodeList;
/**
* Implementation of a Ignite data store to be used by gora.
*
- * @param <K>
- * class to be used for the key
- * @param <T>
- * class to be persisted within the store
+ * @param <K> class to be used for the key
+ * @param <T> class to be persisted within the store
*/
-public class IgniteStore<K,T extends PersistentBase> extends DataStoreBase<K,T> {
-
- protected static final String MOCK_PROPERTY = "ignite.mock";
- protected static final String INSTANCE_NAME_PROPERTY = "ignite.instance";
- protected static final String ZOOKEEPERS_NAME_PROPERTY = "ignite.zookeepers";
- protected static final String USERNAME_PROPERTY = "ignite.user";
- protected static final String PASSWORD_PROPERTY = "ignite.password";
- protected static final String DEFAULT_MAPPING_FILE = "gora-ignite-mapping.xml";
-
- private final static String UNKOWN = "Unknown type ";
-
- private Connector conn;
- private BatchWriter batchWriter;
- private IgniteMapping mapping;
- private Credentials credentials;
- private Encoder encoder;
+public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T> {
public static final Logger LOG = LoggerFactory.getLogger(IgniteStore.class);
+ private static final String PARSE_MAPPING_FILE_KEY = "gora.ignite.mapping.file";
+ private static final String DEFAULT_MAPPING_FILE = "gora-ignite-mapping.xml";
+ private IgniteParameters igniteParameters;
+ private IgniteMapping igniteMapping;
+ private Connection connection;
- public Object fromBytes(Schema schema, byte[] data) throws IOException {
- Schema fromSchema = null;
- if (schema.getType() == Type.UNION) {
- try {
- Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
- int unionIndex = decoder.readIndex();
- List<Schema> possibleTypes = schema.getTypes();
- fromSchema = possibleTypes.get(unionIndex);
- Schema effectiveSchema = possibleTypes.get(unionIndex);
- if (effectiveSchema.getType() == Type.NULL) {
- decoder.readNull();
- return null;
- } else {
- data = decoder.readBytes(null).array();
- }
- } catch (IOException e) {
- LOG.error(e.getMessage());
- throw new GoraException("Error decoding union type: ", e);
- }
- } else {
- fromSchema = schema;
- }
- return fromBytes(encoder, fromSchema, data);
- }
-
- public static Object fromBytes(Encoder encoder, Schema schema, byte data[]) throws IOException {
- switch (schema.getType()) {
- case BOOLEAN:
- return encoder.decodeBoolean(data);
- case DOUBLE:
- return encoder.decodeDouble(data);
- case FLOAT:
- return encoder.decodeFloat(data);
- case INT:
- return encoder.decodeInt(data);
- case LONG:
- return encoder.decodeLong(data);
- case STRING:
- return new Utf8(data);
- case BYTES:
- return ByteBuffer.wrap(data);
- case ENUM:
- return AvroUtils.getEnumValue(schema, encoder.decodeInt(data));
- case ARRAY:
- break;
- case FIXED:
- break;
- case MAP:
- break;
- case NULL:
- break;
- case RECORD:
- break;
- case UNION:
- break;
- default:
- break;
- }
- throw new IllegalArgumentException(UNKOWN + schema.getType());
-
- }
-
- private static byte[] getBytes(Text text) {
- byte[] bytes = text.getBytes();
- if (bytes.length != text.getLength()) {
- bytes = new byte[text.getLength()];
- System.arraycopy(text.getBytes(), 0, bytes, 0, bytes.length);
- }
- return bytes;
- }
-
- public K fromBytes(Class<K> clazz, byte[] val) {
- return fromBytes(encoder, clazz, val);
- }
+ @Override
+ public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) throws GoraException {
- @SuppressWarnings("unchecked")
- public static <K> K fromBytes(Encoder encoder, Class<K> clazz, byte[] val) {
try {
- if (clazz.equals(Byte.TYPE) || clazz.equals(Byte.class)) {
- return (K) Byte.valueOf(encoder.decodeByte(val));
- } else if (clazz.equals(Boolean.TYPE) || clazz.equals(Boolean.class)) {
- return (K) Boolean.valueOf(encoder.decodeBoolean(val));
- } else if (clazz.equals(Short.TYPE) || clazz.equals(Short.class)) {
- return (K) Short.valueOf(encoder.decodeShort(val));
- } else if (clazz.equals(Integer.TYPE) || clazz.equals(Integer.class)) {
- return (K) Integer.valueOf(encoder.decodeInt(val));
- } else if (clazz.equals(Long.TYPE) || clazz.equals(Long.class)) {
- return (K) Long.valueOf(encoder.decodeLong(val));
- } else if (clazz.equals(Float.TYPE) || clazz.equals(Float.class)) {
- return (K) Float.valueOf(encoder.decodeFloat(val));
- } else if (clazz.equals(Double.TYPE) || clazz.equals(Double.class)) {
- return (K) Double.valueOf(encoder.decodeDouble(val));
- } else if (clazz.equals(String.class)) {
- return (K) new String(val, "UTF-8");
- } else if (clazz.equals(Utf8.class)) {
- return (K) new Utf8(val);
- }
-
- throw new IllegalArgumentException(UNKOWN + clazz.getName());
- } catch (IOException ioe) {
- LOG.error(ioe.getMessage());
- throw new RuntimeException(ioe);
+ super.initialize(keyClass, persistentClass, properties);
+ IgniteMappingBuilder builder = new IgniteMappingBuilder(this);
+ builder.readMappingFile(getConf().get(PARSE_MAPPING_FILE_KEY, DEFAULT_MAPPING_FILE));
+ igniteMapping = builder.getIgniteMapping();
+ igniteParameters = IgniteParameters.load(properties, conf);
+ connection = acquiereConnection();
+ LOG.info("Ignite store was successfully initialized");
+ } catch (ClassNotFoundException | SQLException ex) {
+ LOG.error("Error while initializing Ignite store", ex);
+ throw new GoraException(ex);
}
}
- private static byte[] copyIfNeeded(byte b[], int offset, int len) {
- if (len != b.length || offset != 0) {
- byte[] copy = new byte[len];
- System.arraycopy(b, offset, copy, 0, copy.length);
- b = copy;
+ private Connection acquiereConnection() throws ClassNotFoundException, SQLException {
+ Class.forName("org.apache.ignite.IgniteJdbcThinDriver");
+ StringBuilder urlBuilder = new StringBuilder();
+ urlBuilder.append("jdbc:ignite:thin://");
+ urlBuilder.append(igniteParameters.getHost());
+ if (igniteParameters.getPort() != null) {
+ urlBuilder.append(":" + igniteParameters.getPort());
}
- return b;
- }
-
- public byte[] toBytes(Schema toSchema, Object o) {
- if (toSchema != null && toSchema.getType() == Type.UNION) {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- org.apache.avro.io.BinaryEncoder avroEncoder = EncoderFactory.get().binaryEncoder(baos, null);
- int unionIndex = 0;
- try {
- if (o == null) {
- unionIndex = firstNullSchemaTypeIndex(toSchema);
- avroEncoder.writeIndex(unionIndex);
- avroEncoder.writeNull();
- } else {
- unionIndex = firstNotNullSchemaTypeIndex(toSchema);
- avroEncoder.writeIndex(unionIndex);
- avroEncoder.writeBytes(toBytes(o));
- }
- avroEncoder.flush();
- return baos.toByteArray();
- } catch (IOException e) {
- LOG.error(e.getMessage());
- return toBytes(o);
- }
- } else {
- return toBytes(o);
+ if (igniteParameters.getSchema() != null) {
+ urlBuilder.append("/" + igniteParameters.getSchema());
}
- }
-
- private int firstNullSchemaTypeIndex(Schema toSchema) {
- List<Schema> possibleTypes = toSchema.getTypes();
- int unionIndex = 0;
- for (int i = 0; i < possibleTypes.size(); i++ ) {
- Type pType = possibleTypes.get(i).getType();
- if (pType == Type.NULL) { // FIXME HUGE kludge to pass tests
- unionIndex = i; break;
- }
+ if (igniteParameters.getUser() != null) {
+ urlBuilder.append(";" + igniteParameters.getUser());
}
- return unionIndex;
- }
-
- private int firstNotNullSchemaTypeIndex(Schema toSchema) {
- List<Schema> possibleTypes = toSchema.getTypes();
- int unionIndex = 0;
- for (int i = 0; i < possibleTypes.size(); i++ ) {
- Type pType = possibleTypes.get(i).getType();
- if (pType != Type.NULL) { // FIXME HUGE kludge to pass tests
- unionIndex = i; break;
- }
+ if (igniteParameters.getPassword() != null) {
+ urlBuilder.append(";" + igniteParameters.getPassword());
}
- return unionIndex;
- }
-
- public byte[] toBytes(Object o) {
- return toBytes(encoder, o);
- }
-
- public static byte[] toBytes(Encoder encoder, Object o) {
-
- try {
- if (o instanceof String) {
- return ((String) o).getBytes("UTF-8");
- } else if (o instanceof Utf8) {
- return copyIfNeeded(((Utf8) o).getBytes(), 0, ((Utf8) o).getByteLength());
- } else if (o instanceof ByteBuffer) {
- return copyIfNeeded(((ByteBuffer) o).array(), ((ByteBuffer) o).arrayOffset() + ((ByteBuffer) o).position(), ((ByteBuffer) o).remaining());
- } else if (o instanceof Long) {
- return encoder.encodeLong((Long) o);
- } else if (o instanceof Integer) {
- return encoder.encodeInt((Integer) o);
- } else if (o instanceof Short) {
- return encoder.encodeShort((Short) o);
- } else if (o instanceof Byte) {
- return encoder.encodeByte((Byte) o);
- } else if (o instanceof Boolean) {
- return encoder.encodeBoolean((Boolean) o);
- } else if (o instanceof Float) {
- return encoder.encodeFloat((Float) o);
- } else if (o instanceof Double) {
- return encoder.encodeDouble((Double) o);
- } else if (o instanceof Enum) {
- return encoder.encodeInt(((Enum<?>) o).ordinal());
- }
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
+ if (igniteParameters.getAdditionalConfigurations() != null) {
+ urlBuilder.append(igniteParameters.getAdditionalConfigurations());
}
-
- throw new IllegalArgumentException(UNKOWN + o.getClass().getName());
- }
-
- private BatchWriter getBatchWriter() throws IOException {
- if (batchWriter == null)
- try {
- BatchWriterConfig batchWriterConfig = new BatchWriterConfig();
- batchWriterConfig.setMaxMemory(10000000);
- batchWriterConfig.setMaxLatency(60000L, TimeUnit.MILLISECONDS);
- batchWriterConfig.setMaxWriteThreads(4);
- batchWriter = conn.createBatchWriter(mapping.tableName, batchWriterConfig);
- } catch (TableNotFoundException e) {
- throw new IOException(e);
- }
- return batchWriter;
+ Connection conn = DriverManager.getConnection(urlBuilder.toString());
+ return conn;
}
- /**
- * Initialize the data store by reading the credentials, setting the client's properties up and
- * reading the mapping file. Initialize is called when then the call to
- * {@link org.apache.gora.store.DataStoreFactory#createDataStore} is made.
- *
- * @param keyClass
- * @param persistentClass
- * @param properties
- */
@Override
- public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) throws GoraException {
- super.initialize(keyClass, persistentClass, properties);
-
- try {
-
- String mock = DataStoreFactory.findProperty(properties, this, MOCK_PROPERTY, null);
- String mappingFile = DataStoreFactory.getMappingFile(properties, this, DEFAULT_MAPPING_FILE);
- String user = DataStoreFactory.findProperty(properties, this, USERNAME_PROPERTY, null);
- String password = DataStoreFactory.findProperty(properties, this, PASSWORD_PROPERTY, null);
-
- mapping = readMapping(mappingFile);
-
- if (mapping.encoder == null || "".equals(mapping.encoder)) {
- encoder = new BinaryEncoder();
- } else {
- encoder = (Encoder) getClass().getClassLoader().loadClass(mapping.encoder).newInstance();
- }
-
- AuthenticationToken token = new PasswordToken(password);
- if (mock == null || !mock.equals("true")) {
- String instance = DataStoreFactory.findProperty(properties, this, INSTANCE_NAME_PROPERTY, null);
- String zookeepers = DataStoreFactory.findProperty(properties, this, ZOOKEEPERS_NAME_PROPERTY, null);
- conn = new ZooKeeperInstance(instance, zookeepers).getConnector(user, token);
- } else {
- conn = new MockInstance().getConnector(user, token);
- }
- credentials = new Credentials(user, token);
-
- if (autoCreateSchema && !schemaExists())
- createSchema();
-
- } catch (IOException | InstantiationException | IllegalAccessException |
- ClassNotFoundException | IgniteException | IgniteSecurityException e) {
- throw new GoraException(e);
- }
- }
-
- protected IgniteMapping readMapping(String filename) throws IOException {
- try {
-
- IgniteMapping mapping = new IgniteMapping();
-
- DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder();
- Document dom = db.parse(getClass().getClassLoader().getResourceAsStream(filename));
-
- Element root = dom.getDocumentElement();
-
- NodeList nl = root.getElementsByTagName("class");
- for (int i = 0; i < nl.getLength(); i++) {
-
- Element classElement = (Element) nl.item(i);
- if (classElement.getAttribute("keyClass").equals(keyClass.getCanonicalName())
- && classElement.getAttribute("name").equals(persistentClass.getCanonicalName())) {
-
- mapping.tableName = getSchemaName(classElement.getAttribute("table"), persistentClass);
- mapping.encoder = classElement.getAttribute("encoder");
-
- NodeList fields = classElement.getElementsByTagName("field");
- for (int j = 0; j < fields.getLength(); j++) {
- Element fieldElement = (Element) fields.item(j);
-
- String name = fieldElement.getAttribute("name");
- String family = fieldElement.getAttribute("family");
- String qualifier = fieldElement.getAttribute("qualifier");
- if ("".equals(qualifier))
- qualifier = null;
-
- Pair<Text,Text> col = new Pair<>(new Text(family), qualifier == null ? null : new Text(qualifier));
- mapping.fieldMap.put(name, col);
- mapping.columnMap.put(col, name);
- }
- }
-
- }
-
- if (mapping.tableName == null) {
- throw new GoraException("Please define the ignite 'table' name mapping in " + filename + " for " + persistentClass.getCanonicalName());
- }
-
- nl = root.getElementsByTagName("table");
- for (int i = 0; i < nl.getLength(); i++) {
- Element tableElement = (Element) nl.item(i);
- if (tableElement.getAttribute("name").equals(mapping.tableName)) {
- NodeList configs = tableElement.getElementsByTagName("config");
- for (int j = 0; j < configs.getLength(); j++) {
- Element configElement = (Element) configs.item(j);
- String key = configElement.getAttribute("key");
- String val = configElement.getAttribute("value");
- mapping.tableConfig.put(key, val);
- }
- }
- }
-
- return mapping;
- } catch (Exception ex) {
- throw new IOException("Unable to read " + filename, ex);
- }
-
+ public String getSchemaName() {
+ return igniteMapping.getTableName();
}
@Override
- public String getSchemaName() {
- return mapping.tableName;
+ public String getSchemaName(final String mappingSchemaName,
+ final Class<?> persistentClass) {
+ return super.getSchemaName(mappingSchemaName, persistentClass);
}
@Override
public void createSchema() throws GoraException {
- try {
- conn.tableOperations().create(mapping.tableName);
- Set<Entry<String,String>> es = mapping.tableConfig.entrySet();
- for (Entry<String,String> entry : es) {
- conn.tableOperations().setProperty(mapping.tableName, entry.getKey(), entry.getValue());
- }
-
- } catch (TableExistsException e) {
- LOG.debug(e.getMessage(), e);
- // Assume this is not an error
- } catch (IgniteException | IgniteSecurityException e) {
- throw new GoraException(e);
- }
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
public void deleteSchema() throws GoraException {
- try {
- if (batchWriter != null)
- batchWriter.close();
- batchWriter = null;
- conn.tableOperations().delete(mapping.tableName);
- } catch (TableNotFoundException e) {
- // Ignore. Delete a non existant schema is a success
- } catch (IgniteException | IgniteSecurityException e) {
- throw new GoraException(e);
- }
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
public boolean schemaExists() throws GoraException {
- try {
- return conn.tableOperations().exists(mapping.tableName);
- } catch (Exception e) {
- throw new GoraException(e);
- }
- }
-
- public ByteSequence populate(Iterator<Entry<Key,Value>> iter, T persistent) throws IOException {
- ByteSequence row = null;
-
- Map<Utf8, Object> currentMap = null;
- List currentArray = null;
- Text currentFam = null;
- int currentPos = 0;
- Schema currentSchema = null;
- Field currentField = null;
-
- BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(new byte[0], null);
-
- while (iter.hasNext()) {
- Entry<Key,Value> entry = iter.next();
-
- if (row == null) {
- row = entry.getKey().getRowData();
- }
- byte[] val = entry.getValue().get();
-
- Field field = fieldMap.get(getFieldName(entry));
-
- if (currentMap != null) {
- if (currentFam.equals(entry.getKey().getColumnFamily())) {
- currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
- fromBytes(currentSchema, entry.getValue().get()));
- continue;
- } else {
- persistent.put(currentPos, currentMap);
- currentMap = null;
- }
- } else if (currentArray != null) {
- if (currentFam.equals(entry.getKey().getColumnFamily())) {
- currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
- continue;
- } else {
- persistent.put(currentPos, new GenericData.Array<T>(currentField.schema(), currentArray));
- currentArray = null;
- }
- }
-
- switch (field.schema().getType()) {
- case MAP: // first entry only. Next are handled above on the next loop
- currentMap = new DirtyMapWrapper<>(new HashMap<Utf8, Object>());
- currentPos = field.pos();
- currentFam = entry.getKey().getColumnFamily();
- currentSchema = field.schema().getValueType();
-
- currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
- fromBytes(currentSchema, entry.getValue().get()));
- break;
- case ARRAY:
- currentArray = new DirtyListWrapper<>(new ArrayList<>());
- currentPos = field.pos();
- currentFam = entry.getKey().getColumnFamily();
- currentSchema = field.schema().getElementType();
- currentField = field;
-
- currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
-
- break;
- case UNION:// default value of null acts like union with null
- Schema effectiveSchema = field.schema().getTypes()
- .get(firstNotNullSchemaTypeIndex(field.schema()));
- // map and array were coded without union index so need to be read the same way
- if (effectiveSchema.getType() == Type.ARRAY) {
- currentArray = new DirtyListWrapper<>(new ArrayList<>());
- currentPos = field.pos();
- currentFam = entry.getKey().getColumnFamily();
- currentSchema = field.schema().getElementType();
- currentField = field;
-
- currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
- break;
- }
- else if (effectiveSchema.getType() == Type.MAP) {
- currentMap = new DirtyMapWrapper<>(new HashMap<Utf8, Object>());
- currentPos = field.pos();
- currentFam = entry.getKey().getColumnFamily();
- currentSchema = effectiveSchema.getValueType();
-
- currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
- fromBytes(currentSchema, entry.getValue().get()));
- break;
- }
- // continue like a regular top-level union
- case RECORD:
- SpecificDatumReader<?> reader = new SpecificDatumReader<Schema>(field.schema());
- persistent.put(field.pos(), reader.read(null, DecoderFactory.get().binaryDecoder(val, decoder)));
- break;
- default:
- persistent.put(field.pos(), fromBytes(field.schema(), entry.getValue().get()));
- }
- }
-
- if (currentMap != null) {
- persistent.put(currentPos, currentMap);
- } else if (currentArray != null) {
- persistent.put(currentPos, new GenericData.Array<T>(currentField.schema(), currentArray));
- }
-
- persistent.clearDirty();
-
- return row;
- }
-
- /**
- * Retrieve field name from entry.
- * @param entry The Key-Value entry
- * @return String The field name
- */
- private String getFieldName(Entry<Key, Value> entry) {
- String fieldName = mapping.columnMap.get(new Pair<>(entry.getKey().getColumnFamily(),
- entry.getKey().getColumnQualifier()));
- if (fieldName == null) {
- fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), null));
- }
- return fieldName;
- }
-
- private void setFetchColumns(Scanner scanner, String[] fields) {
- fields = getFieldsToQuery(fields);
- for (String field : fields) {
- Pair<Text,Text> col = mapping.fieldMap.get(field);
- if (col != null) {
- if (col.getSecond() == null) {
- scanner.fetchColumnFamily(col.getFirst());
- } else {
- scanner.fetchColumn(col.getFirst(), col.getSecond());
- }
+ boolean exists = false;
+ try (Statement stmt = connection.createStatement()) {
+ MessageFormat messageFormat = new MessageFormat("select * from {0} limit 0", Locale.getDefault());
+ ResultSet executeQuery = stmt.executeQuery(messageFormat.format(igniteMapping.getTableName()));
+ executeQuery.close();
+ exists = true;
+ } catch (SQLException ex) {
+ /**
+ * a 42000 error code is thrown by Ignite when a non-existent table
+ * queried. More details:
+ * https://apacheignite-sql.readme.io/docs/jdbc-error-codes
+ */
+ if (ex.getSQLState() != null && ex.getSQLState().equals("42000")) {
+ exists = false;
} else {
- LOG.error("Mapping not found for field: {}", field);
+ throw new GoraException(ex);
}
}
+ return exists;
}
@Override
public T get(K key, String[] fields) throws GoraException {
- try {
- // TODO make isolated scanner optional?
- Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
- Range rowRange = new Range(new Text(toBytes(key)));
-
- scanner.setRange(rowRange);
- setFetchColumns(scanner, fields);
-
- T persistent = newPersistent();
- ByteSequence row = populate(scanner.iterator(), persistent);
- if (row == null)
- return null;
- return persistent;
- } catch (Exception e) {
- throw new GoraException(e);
- }
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
- public void put(K key, T val) throws GoraException {
-
- try{
- Mutation m = new Mutation(new Text(toBytes(key)));
-
- Schema schema = val.getSchema();
- List<Field> fields = schema.getFields();
- int count = 0;
-
- for (int i = 0; i < fields.size(); i++) {
- if (!val.isDirty(i)) {
- continue;
- }
- Field field = fields.get(i);
-
- Object o = val.get(field.pos());
-
- Pair<Text,Text> col = mapping.fieldMap.get(field.name());
-
- if (col == null) {
- throw new GoraException("Please define the gora to ignite mapping for field " + field.name());
- }
-
- switch (field.schema().getType()) {
- case MAP:
- count = putMap(m, count, field.schema().getValueType(), o, col, field.name());
- break;
- case ARRAY:
- count = putArray(m, count, o, col, field.name());
- break;
- case UNION: // default value of null acts like union with null
- Schema effectiveSchema = field.schema().getTypes()
- .get(firstNotNullSchemaTypeIndex(field.schema()));
- // map and array need to compute qualifier
- if (effectiveSchema.getType() == Type.ARRAY) {
- count = putArray(m, count, o, col, field.name());
- break;
- }
- else if (effectiveSchema.getType() == Type.MAP) {
- count = putMap(m, count, effectiveSchema.getValueType(), o, col, field.name());
- break;
- }
- // continue like a regular top-level union
- case RECORD:
- final SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(field.schema());
- final byte[] byteData = IOUtils.serialize(writer,o);
- m.put(col.getFirst(), col.getSecond(), new Value(byteData));
- count++;
- break;
- default:
- m.put(col.getFirst(), col.getSecond(), new Value(toBytes(o)));
- count++;
- }
-
- }
-
- if (count > 0)
- try {
- getBatchWriter().addMutation(m);
- } catch (MutationsRejectedException e) {
- LOG.error(e.getMessage(), e);
- }
- } catch (GoraException e) {
- throw e;
- } catch (Exception e) {
- throw new GoraException(e);
- }
- }
-
- private int putMap(Mutation m, int count, Schema valueType, Object o, Pair<Text, Text> col, String fieldName) throws GoraException {
-
- // First of all we delete map field on ignite store
- Text rowKey = new Text(m.getRow());
- Query<K, T> query = newQuery();
- query.setFields(fieldName);
- query.setStartKey((K)rowKey.toString());
- query.setEndKey((K)rowKey.toString());
- deleteByQuery(query);
- flush();
- if (o == null){
- return 0;
- }
-
- Set<?> es = ((Map<?, ?>)o).entrySet();
- for (Object entry : es) {
- Object mapKey = ((Entry<?, ?>) entry).getKey();
- Object mapVal = ((Entry<?, ?>) entry).getValue();
- if ((o instanceof DirtyMapWrapper && ((DirtyMapWrapper<?, ?>)o).isDirty())
- || !(o instanceof DirtyMapWrapper)) {
- m.put(col.getFirst(), new Text(toBytes(mapKey)), new Value(toBytes(valueType, mapVal)));
- count++;
- }
- // TODO map value deletion
- }
- return count;
- }
-
- private int putArray(Mutation m, int count, Object o, Pair<Text, Text> col, String fieldName) throws GoraException {
-
- // First of all we delete array field on ignite store
- Text rowKey = new Text(m.getRow());
- Query<K, T> query = newQuery();
- query.setFields(fieldName);
- query.setStartKey((K)rowKey.toString());
- query.setEndKey((K)rowKey.toString());
- deleteByQuery(query);
- flush();
- if (o == null){
- return 0;
- }
-
- List<?> array = (List<?>) o; // both GenericArray and DirtyListWrapper
- int j = 0;
- for (Object item : array) {
- m.put(col.getFirst(), new Text(toBytes(j++)), new Value(toBytes(item)));
- count++;
- }
- return count;
+ public void put(K key, T obj) throws GoraException {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
public boolean delete(K key) throws GoraException {
- Query<K,T> q = newQuery();
- q.setKey(key);
- return deleteByQuery(q) > 0;
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
- public long deleteByQuery(Query<K,T> query) throws GoraException {
- try {
- Scanner scanner = createScanner(query);
- // add iterator that drops values on the server side
- scanner.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class));
- RowIterator iterator = new RowIterator(scanner.iterator());
-
- long count = 0;
-
- while (iterator.hasNext()) {
- Iterator<Entry<Key,Value>> row = iterator.next();
- Mutation m = null;
- while (row.hasNext()) {
- Entry<Key,Value> entry = row.next();
- Key key = entry.getKey();
- if (m == null)
- m = new Mutation(key.getRow());
- // TODO optimize to avoid continually creating column vis? prob does not matter for empty
- m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp());
- }
- getBatchWriter().addMutation(m);
- count++;
- }
-
- return count;
- } catch (Exception e) {
- throw new GoraException(e);
- }
+ public long deleteByQuery(Query<K, T> query) throws GoraException {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
- private Range createRange(Query<K,T> query) {
- Text startRow = null;
- Text endRow = null;
-
- if (query.getStartKey() != null)
- startRow = new Text(toBytes(query.getStartKey()));
-
- if (query.getEndKey() != null)
- endRow = new Text(toBytes(query.getEndKey()));
-
- return new Range(startRow, true, endRow, true);
-
- }
-
- private Scanner createScanner(Query<K,T> query) throws TableNotFoundException {
- // TODO make isolated scanner optional?
- Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
- setFetchColumns(scanner, query.getFields());
-
- scanner.setRange(createRange(query));
-
- if (query.getStartTime() != -1 || query.getEndTime() != -1) {
- IteratorSetting is = new IteratorSetting(30, TimestampFilter.class);
- if (query.getStartTime() != -1)
- TimestampFilter.setStart(is, query.getStartTime(), true);
- if (query.getEndTime() != -1)
- TimestampFilter.setEnd(is, query.getEndTime(), true);
-
- scanner.addScanIterator(is);
- }
-
- return scanner;
- }
-
- /**
- * Execute the query and return the result.
- */
@Override
- public Result<K,T> execute(Query<K,T> query) throws GoraException {
- try {
- Scanner scanner = createScanner(query);
- return new IgniteResult<>(this, query, scanner);
- } catch (TableNotFoundException e) {
- throw new GoraException(e) ;
- }
+ public Result<K, T> execute(Query<K, T> query) throws GoraException {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
- public Query<K,T> newQuery() {
- return new IgniteQuery<>(this);
- }
-
- Text pad(Text key, int bytes) {
- if (key.getLength() < bytes)
- key = new Text(key);
-
- while (key.getLength() < bytes) {
- key.append(new byte[] {0}, 0, 1);
- }
-
- return key;
+ public Query<K, T> newQuery() {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
- public List<PartitionQuery<K,T>> getPartitions(Query<K,T> query) throws GoraException {
- try {
- TabletLocator tl;
- if (conn instanceof MockConnector)
- tl = new MockTabletLocator();
- else
- tl = TabletLocator.getLocator(new ClientContext(conn.getInstance(), credentials, IgniteConfiguration.getTableConfiguration(conn, Tables.getTableId(conn.getInstance(), mapping.tableName))), new Text(Tables.getTableId(conn.getInstance(), mapping.tableName)));
-
- Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
-
- tl.invalidateCache();
- while (tl.binRanges(new ClientContext(conn.getInstance(), credentials, IgniteConfiguration.getTableConfiguration(conn, Tables.getTableId(conn.getInstance(), mapping.tableName))), Collections.singletonList(createRange(query)), binnedRanges).size() > 0) {
- // TODO log?
- if (!Tables.exists(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName)))
- throw new TableDeletedException(Tables.getTableId(conn.getInstance(), mapping.tableName));
- else if (Tables.getTableState(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName)) == TableState.OFFLINE)
- throw new TableOfflineException(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName));
- UtilWaitThread.sleep(100);
- tl.invalidateCache();
- }
-
- List<PartitionQuery<K,T>> ret = new ArrayList<>();
-
- Text startRow = null;
- Text endRow = null;
- if (query.getStartKey() != null)
- startRow = new Text(toBytes(query.getStartKey()));
- if (query.getEndKey() != null)
- endRow = new Text(toBytes(query.getEndKey()));
-
- //hadoop expects hostnames, ignite keeps track of IPs... so need to convert
- HashMap<String,String> hostNameCache = new HashMap<>();
-
- for (Entry<String,Map<KeyExtent,List<Range>>> entry : binnedRanges.entrySet()) {
- String ip = entry.getKey().split(":", 2)[0];
- String location = hostNameCache.get(ip);
- if (location == null) {
- InetAddress inetAddress = InetAddress.getByName(ip);
- location = inetAddress.getHostName();
- hostNameCache.put(ip, location);
- }
-
- Map<KeyExtent,List<Range>> tablets = entry.getValue();
- for (KeyExtent ke : tablets.keySet()) {
-
- K startKey = null;
- if (startRow == null || !ke.contains(startRow)) {
- if (ke.getPrevEndRow() != null) {
- startKey = followingKey(encoder, getKeyClass(), getBytes(ke.getPrevEndRow()));
- }
- } else {
- startKey = fromBytes(getKeyClass(), getBytes(startRow));
- }
-
- K endKey = null;
- if (endRow == null || !ke.contains(endRow)) {
- if (ke.getEndRow() != null)
- endKey = lastPossibleKey(encoder, getKeyClass(), getBytes(ke.getEndRow()));
- } else {
- endKey = fromBytes(getKeyClass(), getBytes(endRow));
- }
-
- PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query, startKey, endKey, location);
- pqi.setConf(getConf());
- ret.add(pqi);
- }
- }
-
- return ret;
- } catch (Exception e) {
- throw new GoraException(e);
- }
-
- }
-
- static <K> K lastPossibleKey(Encoder encoder, Class<K> clazz, byte[] er) {
-
- if (clazz.equals(Byte.TYPE) || clazz.equals(Byte.class)) {
- throw new UnsupportedOperationException();
- } else if (clazz.equals(Boolean.TYPE) || clazz.equals(Boolean.class)) {
- throw new UnsupportedOperationException();
- } else if (clazz.equals(Short.TYPE) || clazz.equals(Short.class)) {
- return fromBytes(encoder, clazz, encoder.lastPossibleKey(2, er));
- } else if (clazz.equals(Integer.TYPE) || clazz.equals(Integer.class)) {
- return fromBytes(encoder, clazz, encoder.lastPossibleKey(4, er));
- } else if (clazz.equals(Long.TYPE) || clazz.equals(Long.class)) {
- return fromBytes(encoder, clazz, encoder.lastPossibleKey(8, er));
- } else if (clazz.equals(Float.TYPE) || clazz.equals(Float.class)) {
- return fromBytes(encoder, clazz, encoder.lastPossibleKey(4, er));
- } else if (clazz.equals(Double.TYPE) || clazz.equals(Double.class)) {
- return fromBytes(encoder, clazz, encoder.lastPossibleKey(8, er));
- } else if (clazz.equals(String.class)) {
- throw new UnsupportedOperationException();
- } else if (clazz.equals(Utf8.class)) {
- return fromBytes(encoder, clazz, er);
- }
-
- throw new IllegalArgumentException(UNKOWN + clazz.getName());
- }
-
- @SuppressWarnings("unchecked")
- static <K> K followingKey(Encoder encoder, Class<K> clazz, byte[] per) {
-
- if (clazz.equals(Byte.TYPE) || clazz.equals(Byte.class)) {
- return (K) Byte.valueOf(encoder.followingKey(1, per)[0]);
- } else if (clazz.equals(Boolean.TYPE) || clazz.equals(Boolean.class)) {
- throw new UnsupportedOperationException();
- } else if (clazz.equals(Short.TYPE) || clazz.equals(Short.class)) {
- return fromBytes(encoder, clazz, encoder.followingKey(2, per));
- } else if (clazz.equals(Integer.TYPE) || clazz.equals(Integer.class)) {
- return fromBytes(encoder, clazz, encoder.followingKey(4, per));
- } else if (clazz.equals(Long.TYPE) || clazz.equals(Long.class)) {
- return fromBytes(encoder, clazz, encoder.followingKey(8, per));
- } else if (clazz.equals(Float.TYPE) || clazz.equals(Float.class)) {
- return fromBytes(encoder, clazz, encoder.followingKey(4, per));
- } else if (clazz.equals(Double.TYPE) || clazz.equals(Double.class)) {
- return fromBytes(encoder, clazz, encoder.followingKey(8, per));
- } else if (clazz.equals(String.class)) {
- throw new UnsupportedOperationException();
- } else if (clazz.equals(Utf8.class)) {
- return fromBytes(encoder, clazz, Arrays.copyOf(per, per.length + 1));
- }
-
- throw new IllegalArgumentException(UNKOWN + clazz.getName());
+ public List<PartitionQuery<K, T>> getPartitions(Query<K, T> query) throws IOException {
+ throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
@Override
public void flush() throws GoraException {
try {
- if (batchWriter != null) {
- batchWriter.flush();
- }
+ connection.commit();
} catch (Exception e) {
throw new GoraException(e);
}
@@ -1023,12 +183,11 @@ public class IgniteStore<K,T extends PersistentBase> extends DataStoreBase<K,T>
@Override
public void close() {
try {
- if (batchWriter != null) {
- batchWriter.close();
- batchWriter = null;
- }
- } catch (MutationsRejectedException e) {
- LOG.error(e.getMessage(), e);
+ connection.close();
+ LOG.info("Ignite datastore destroyed successfully.");
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
}
}
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java b/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java
deleted file mode 100644
index 97fb46a..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.util;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * It is a implementation of {@link java.io.OutputStream} must always provide at least a method that writes one byte of output.
- */
-public class FixedByteArrayOutputStream extends OutputStream {
-
- private int i;
- byte out[];
-
- public FixedByteArrayOutputStream(byte out[]) {
- this.out = out;
- }
-
- @Override
- public void write(int b) throws IOException {
- out[i++] = (byte) b;
- }
-
- @Override
- public void write(byte b[], int off, int len) throws IOException {
- System.arraycopy(b, off, out, i, len);
- i += len;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java
deleted file mode 100644
index eedb84b..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This package contains Ignite store related util classes.
- */
-package org.apache.gora.ignite.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java b/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
index 1e03717..336d81b 100644
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/GoraIgniteTestDriver.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,57 +17,32 @@
*/
package org.apache.gora.ignite;
-import org.apache.ignite.minicluster.MiniIgniteCluster;
-import org.apache.ignite.minicluster.MiniIgniteConfig;
import org.apache.gora.GoraTestDriver;
import org.apache.gora.ignite.store.IgniteStore;
-import org.junit.Rule;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
/**
- * @author lmcgibbn
+ * Helper class for third part tests using gora-ignite backend.
*
+ * @see GoraTestDriver
*/
public class GoraIgniteTestDriver extends GoraTestDriver {
- private static final Logger LOG = LoggerFactory.getLogger(GoraIgniteTestDriver.class);
- private static MiniIgniteCluster cluster = null;
- private static final String PASSWORD = "drowssap";
+ private Ignite igniteInstance;
- @Rule
- public TemporaryFolder tmpDir = new TemporaryFolder();
-
- public GoraIgniteTestDriver() throws Exception {
+ public GoraIgniteTestDriver() {
super(IgniteStore.class);
}
@Override
- public void setUpClass() throws IOException, InterruptedException {
- log.info("Starting Ignite MiniIgniteCluster...");
- try {
- tmpDir.create();
- MiniIgniteConfig miniCfg = new MiniIgniteConfig(tmpDir.getRoot(), PASSWORD);
- miniCfg.setInstanceName("goraTest");
- miniCfg.setZooKeeperPort(56321);
- cluster = new MiniIgniteCluster(miniCfg);
- cluster.start();
- } catch (Exception e) {
- LOG.error("Error starting Ignite MiniIgniteCluster: {}", e.getMessage());
- // cleanup
- tearDownClass();
- }
+ public void setUpClass() throws Exception {
+ igniteInstance = Ignition.start();
}
@Override
- public void tearDownClass() throws IOException, InterruptedException {
- log.info("Shutting down Ignite MiniIgniteCluster...");
- if (cluster != null) {
- cluster.stop();
- }
- tmpDir.delete();
+ public void tearDownClass() throws Exception {
+ igniteInstance.close();
}
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
deleted file mode 100644
index f536961..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Tests for <code>gora-ignite</code> including
- * the test driver for {@link org.apache.gora.ignite.store.IgniteStoreTest}
- */
-package org.apache.gora.ignite;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java
deleted file mode 100644
index bb77bee..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/store/AuthenticationTokenTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.store;
-
-import java.util.List;
-import java.util.Properties;
-import org.apache.ignite.minicluster.MiniIgniteCluster;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.query.PartitionQuery;
-import org.apache.gora.store.DataStore;
-import org.apache.gora.store.DataStoreFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Tests authentication token is serialized correctly.
- */
-public class AuthenticationTokenTest {
- private static final Logger LOG = LoggerFactory.getLogger(AuthenticationTokenTest.class);
-
- private static final String GORA_DATASTORE =
- DataStoreFactory.GORA + "." + DataStoreFactory.DATASTORE + ".";
- private static final String PASSWORD = "password";
-
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
-
- private MiniIgniteCluster cluster;
- private DataStore<String, Employee> employeeStore;
-
- @Before
- @SuppressWarnings("unchecked")
- public void setUp() throws Exception {
- cluster = new MiniIgniteCluster(temporaryFolder.getRoot(), PASSWORD);
- cluster.start();
-
- Properties properties = DataStoreFactory.createProps();
- properties.setProperty(
- GORA_DATASTORE + IgniteStore.MOCK_PROPERTY,
- "false");
- properties.setProperty(
- GORA_DATASTORE + IgniteStore.INSTANCE_NAME_PROPERTY,
- cluster.getInstanceName());
- properties.setProperty(
- GORA_DATASTORE + IgniteStore.ZOOKEEPERS_NAME_PROPERTY,
- cluster.getZooKeepers());
- properties.setProperty(
- GORA_DATASTORE + IgniteStore.PASSWORD_PROPERTY,
- PASSWORD);
-
- employeeStore = DataStoreFactory.createDataStore(
- IgniteStore.class,
- String.class,
- Employee.class,
- new Configuration(),
- properties);
- }
-
- @After
- public void tearDown() throws Exception {
- cluster.stop();
- }
-
- @Test
- public void testAuthenticationTokenIsSerializedCorrectly() throws Exception {
- List<PartitionQuery<String, Employee>> partitions =
- employeeStore.getPartitions(employeeStore.newQuery());
- LOG.debug("partitions {}", partitions);
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java
deleted file mode 100644
index b42d344..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/store/IgniteStoreTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.store;
-
-import java.io.IOException;
-
-import org.apache.gora.ignite.GoraIgniteTestDriver;
-import org.apache.gora.examples.generated.Employee;
-import org.apache.gora.examples.generated.WebPage;
-import org.apache.gora.store.DataStore;
-import org.apache.gora.store.DataStoreFactory;
-import org.apache.gora.store.DataStoreTestBase;
-import static org.apache.gora.store.DataStoreTestBase.log;
-import org.apache.gora.store.DataStoreTestUtil;
-import static org.apache.gora.store.DataStoreTestUtil.testResultSize;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Tests extending {@link org.apache.gora.store.DataStoreTestBase}
- * which run the base JUnit test suite for Gora.
- */
-public class IgniteStoreTest extends DataStoreTestBase {
-
- static {
- try {
- setTestDriver(new GoraIgniteTestDriver());
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
- }
-
- public GoraIgniteTestDriver getTestDriver() {
- return (GoraIgniteTestDriver) testDriver;
- }
-
- //Until GORA-66 is resolved this test will always fail, so
- //do not run it
- @Ignore("skipped until GORA-66 is resolved")
- @Override
- public void testDeleteByQueryFields() throws IOException {
- }
-
- @Test
- @Ignore("Ignite does not support Result#size() without limit set")
- @Override
- public void testResultSize() throws Exception {
- }
-
- @Test
- @Ignore("Ignite does not support Result#size() without limit set")
- @Override
- public void testResultSizeStartKey() throws Exception {
- }
-
- @Ignore("Ignite does not support Result#size() without limit set")
- @Override
- public void testResultSizeEndKey() throws Exception {
- }
-
- @Test
- @Ignore("Ignite does not support Result#size() without limit set")
- @Override
- public void testResultSizeKeyRange() throws Exception {
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java
deleted file mode 100644
index 1c77656..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/store/PartitionTest.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.store;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.gora.ignite.encoders.Encoder;
-import org.apache.gora.ignite.encoders.SignedBinaryEncoder;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- *
- */
-public class PartitionTest {
- // TODO test more types
-
- private static Encoder encoder = new SignedBinaryEncoder();
-
- static long encl(long l) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutputStream dos = new DataOutputStream(baos);
- try {
- dos.writeLong(l);
- dos.flush();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return encoder.decodeLong(baos.toByteArray());
- }
-
- @Test
- public void test1() throws IOException {
- assertEquals(encl(0x006f000000000000l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {0x00, 0x6f}));
- assertEquals(encl(1l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {0, 0, 0, 0, 0, 0, 0, 0}));
- assertEquals(encl(0x106f000000000001l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {0x10, 0x6f, 0, 0, 0, 0, 0, 0}));
- assertEquals(
- encl(-1l),
- (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
- (byte) 0xff,
- (byte) 0xfe}));
-
- assertEquals(encl(0x8000000000000001l), (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {(byte) 0x80, 0, 0, 0, 0, 0, 0, 0}));
- assertEquals(
- encl(0x8000000000000000l),
- (long) IgniteStore.followingKey(encoder, Long.class, new byte[] {(byte) 0x7f, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff,
- (byte) 0xff,
- (byte) 0xff}));
-
-
- try {
- IgniteStore.followingKey(encoder, Long.class,
- new byte[] {(byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff});
- assertTrue(false);
- } catch (IllegalArgumentException iea) {
-
- }
- }
-
- @Test
- public void test2() throws IOException {
- assertEquals(encl(0x00ffffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {0x01}));
- assertEquals(encl(0x006effffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {0x00, 0x6f}));
- assertEquals(encl(0xff6effffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0xff, 0x6f}));
- assertEquals(encl(0xfffeffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0xff, (byte) 0xff}));
- assertEquals(encl(0l), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0, 0, 0, 0, 0, 0, 0, 0}));
-
- assertEquals(encl(0x7effffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0x7f}));
- assertEquals(encl(0x7fffffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0x80}));
- assertEquals(encl(0x80ffffffffffffffl), (long) IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0x81}));
-
- try {
- IgniteStore.lastPossibleKey(encoder, Long.class, new byte[] {(byte) 0, 0, 0, 0, 0, 0, 0});
- assertTrue(false);
- } catch (IllegalArgumentException iea) {
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/store/TestIgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/TestIgniteStore.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/TestIgniteStore.java
new file mode 100644
index 0000000..2244f0e
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/store/TestIgniteStore.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import org.apache.gora.ignite.GoraIgniteTestDriver;
+import org.apache.gora.store.DataStoreTestBase;
+
+/**
+ * Test case for IgniteStore.
+ */
+public class TestIgniteStore extends DataStoreTestBase {
+
+ static {
+ setTestDriver(new GoraIgniteTestDriver());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
deleted file mode 100644
index a41e085..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This package contains all the unit tests for basic CRUD operations
- * functionality of the Ignite dataStore.
- */
-package org.apache.gora.ignite.store;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java
deleted file mode 100644
index 0d4e5e6..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/util/HexEncoderTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.util;
-
-import java.nio.charset.Charset;
-import org.apache.gora.ignite.encoders.HexEncoder;
-import static org.junit.Assert.assertEquals;
-import org.junit.Test;
-
-/**
- *
- */
-public class HexEncoderTest {
-
- @Test
- public void testByte() {
- HexEncoder encoder = new HexEncoder();
-
- assertEquals("12", new String(encoder.encodeByte((byte) 0x12), Charset.defaultCharset()));
- assertEquals("f2", new String(encoder.encodeByte((byte) 0xf2), Charset.defaultCharset()));
-
- byte b = Byte.MIN_VALUE;
- while (b != Byte.MAX_VALUE) {
- assertEquals(b, encoder.decodeByte(encoder.encodeByte(b)));
- b++;
- }
- }
-
- @Test
- public void testShort() {
- HexEncoder encoder = new HexEncoder();
-
- assertEquals("1234", new String(encoder.encodeShort((short) 0x1234), Charset.defaultCharset()));
- assertEquals("f234", new String(encoder.encodeShort((short) 0xf234), Charset.defaultCharset()));
-
- short s = Short.MIN_VALUE;
- while (s != Short.MAX_VALUE) {
- assertEquals(s, encoder.decodeShort(encoder.encodeShort(s)));
- s++;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java
deleted file mode 100644
index c943e73..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/util/SignedBinaryEncoderTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-
-import org.apache.gora.ignite.encoders.SignedBinaryEncoder;
-import org.apache.hadoop.io.Text;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- *
- */
-public class SignedBinaryEncoderTest {
- @Test
- public void testShort() throws IOException {
- short s = Short.MIN_VALUE;
- Text prev = null;
-
- SignedBinaryEncoder encoder = new SignedBinaryEncoder();
-
- while (true) {
- byte[] enc = encoder.encodeShort(s);
- assertEquals(s, encoder.decodeShort(enc));
- Text current = new Text(enc);
- if (prev != null)
- assertTrue(prev.compareTo(current) < 0);
- prev = current;
- s++;
- if (s == Short.MAX_VALUE)
- break;
- }
- }
-
- private void testInt(int start, int finish) throws IOException {
- int i = start;
- Text prev = null;
-
- SignedBinaryEncoder encoder = new SignedBinaryEncoder();
-
- while (true) {
- byte[] enc = encoder.encodeInt(i);
- assertEquals(i, encoder.decodeInt(enc));
- Text current = new Text(enc);
- if (prev != null)
- assertTrue(prev.compareTo(current) < 0);
- prev = current;
- i++;
- if (i == finish)
- break;
- }
- }
-
- @Test
- public void testInt() throws IOException {
- testInt(Integer.MIN_VALUE, Integer.MIN_VALUE + (1 << 16));
- testInt(-(1 << 15), (1 << 15));
- testInt(Integer.MAX_VALUE - (1 << 16), Integer.MAX_VALUE);
- }
-
- private void testLong(long start, long finish) throws IOException {
- long l = start;
- Text prev = null;
-
- SignedBinaryEncoder encoder = new SignedBinaryEncoder();
-
- while (true) {
- byte[] enc = encoder.encodeLong(l);
- assertEquals(l, encoder.decodeLong(enc));
- Text current = new Text(enc);
- if (prev != null)
- assertTrue(prev.compareTo(current) < 0);
- prev = current;
- l++;
- if (l == finish)
- break;
- }
- }
-
- @Test
- public void testLong() throws IOException {
- testLong(Long.MIN_VALUE, Long.MIN_VALUE + (1 << 16));
- testLong(-(1 << 15), (1 << 15));
- testLong(Long.MAX_VALUE - (1 << 16), Long.MAX_VALUE);
- }
-
- @Test
- public void testDouble() throws IOException {
-
- ArrayList<Double> testData = new ArrayList<>();
- testData.add(Double.NEGATIVE_INFINITY);
- testData.add(Double.MIN_VALUE);
- testData.add(Math.nextUp(Double.NEGATIVE_INFINITY));
- testData.add(Math.pow(10.0, 30.0) * -1.0);
- testData.add(Math.pow(10.0, 30.0));
- testData.add(Math.pow(10.0, -30.0) * -1.0);
- testData.add(Math.pow(10.0, -30.0));
- testData.add(Math.nextAfter(0.0, Double.NEGATIVE_INFINITY));
- testData.add(0.0);
- testData.add(Math.nextAfter(Double.MAX_VALUE, Double.NEGATIVE_INFINITY));
- testData.add(Double.MAX_VALUE);
- testData.add(Double.POSITIVE_INFINITY);
-
- Collections.sort(testData);
-
- SignedBinaryEncoder encoder = new SignedBinaryEncoder();
-
- for (int i = 0; i < testData.size(); i++) {
- byte[] enc = encoder.encodeDouble(testData.get(i));
- assertEquals(testData.get(i), (Double) encoder.decodeDouble(enc));
- if (i > 1) {
- assertTrue("Checking " + testData.get(i) + " > " + testData.get(i - 1),
- new Text(enc).compareTo(new Text(encoder.encodeDouble(testData.get(i - 1)))) > 0);
- }
- }
- }
-
- @Test
- public void testFloat() throws IOException {
-
- ArrayList<Float> testData = new ArrayList<>();
- testData.add(Float.NEGATIVE_INFINITY);
- testData.add(Float.MIN_VALUE);
- testData.add(Math.nextUp(Float.NEGATIVE_INFINITY));
- testData.add((float) Math.pow(10.0f, 30.0f) * -1.0f);
- testData.add((float) Math.pow(10.0f, 30.0f));
- testData.add((float) Math.pow(10.0f, -30.0f) * -1.0f);
- testData.add((float) Math.pow(10.0f, -30.0f));
- testData.add(Math.nextAfter(0.0f, Float.NEGATIVE_INFINITY));
- testData.add(0.0f);
- testData.add(Math.nextAfter(Float.MAX_VALUE, Float.NEGATIVE_INFINITY));
- testData.add(Float.MAX_VALUE);
- testData.add(Float.POSITIVE_INFINITY);
-
- Collections.sort(testData);
-
- SignedBinaryEncoder encoder = new SignedBinaryEncoder();
-
- for (int i = 0; i < testData.size(); i++) {
- byte[] enc = encoder.encodeFloat(testData.get(i));
- assertEquals(testData.get(i), (Float)encoder.decodeFloat(enc));
- if (i > 1) {
- assertTrue("Checking " + testData.get(i) + " > " + testData.get(i - 1),
- new Text(enc).compareTo(new Text(encoder.encodeFloat(testData.get(i - 1)))) > 0);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java
deleted file mode 100644
index afcd619..0000000
--- a/gora-ignite/src/test/java/org/apache/gora/ignite/util/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This package contains all the unit tests for utils of the Ignite dataStore.
- */
-package org.apache.gora.ignite.util;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/resources/gora-accumulo-mapping.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora-accumulo-mapping.xml b/gora-ignite/src/test/resources/gora-accumulo-mapping.xml
deleted file mode 100644
index 4d036c6..0000000
--- a/gora-ignite/src/test/resources/gora-accumulo-mapping.xml
+++ /dev/null
@@ -1,59 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<gora-otd>
- <table name="AccessLog">
- <config key="table.file.compress.blocksize" value="32K"/>
- </table>
-
- <class name="org.apache.gora.tutorial.log.generated.Pageview" keyClass="java.lang.Long" table="AccessLog">
- <field name="url" family="common" qualifier="url"/>
- <field name="timestamp" family="common" qualifier="timestamp"/>
- <field name="ip" family="common" qualifier="ip" />
- <field name="httpMethod" family="http" qualifier="httpMethod"/>
- <field name="httpStatusCode" family="http" qualifier="httpStatusCode"/>
- <field name="responseSize" family="http" qualifier="responseSize"/>
- <field name="referrer" family="misc" qualifier="referrer"/>
- <field name="userAgent" family="misc" qualifier="userAgent"/>
- </class>
-
- <class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee">
- <field name="name" family="info" qualifier="nm"/>
- <field name="dateOfBirth" family="info" qualifier="db"/>
- <field name="ssn" family="info" qualifier="sn"/>
- <field name="salary" family="info" qualifier="sl"/>
- <field name="boss" family="info" qualifier="bs"/>
- <field name="webpage" family="info" qualifier="wp"/>
- </class>
-
- <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage">
- <field name="url" family="common" qualifier="u"/>
- <field name="content" family="content" qualifier="c"/>
- <field name="parsedContent" family="parsedContent"/>
- <field name="outlinks" family="outlinks"/>
- <field name="headers" family="headers"/>
- <field name="metadata" family="common" qualifier="metadata"/>
- <field name="byteData" family="byteData" qualifier="byteData"/>
- <field name="stringData" family="stringData" qualifier="stringData"/>
- </class>
-
- <class name="org.apache.gora.examples.generated.TokenDatum" keyClass="java.lang.String">
- <field name="count" family="common" qualifier="count"/>
- </class>
-</gora-otd>
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/resources/gora-ignite-mapping.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora-ignite-mapping.xml b/gora-ignite/src/test/resources/gora-ignite-mapping.xml
new file mode 100644
index 0000000..3c13774
--- /dev/null
+++ b/gora-ignite/src/test/resources/gora-ignite-mapping.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<gora-otd>
+ <class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee">
+ <primarykey column="ssn"/>
+ <field name="ssn" column="ssn" type="VARCHAR"/>
+ <field name="name" column="name" type="VARCHAR"/>
+ <field name="dateOfBirth" column="dateOfBirth" type="BIGINT"/>
+ <field name="salary" column="salary" type="INT"/>
+ <field name="boss" column="boss" type="VARCHAR"/>
+ <field name="webpage" column="webpage" type="VARCHAR"/>
+ </class>
+
+ <class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage">
+ <primarykey column="url"/>
+ <field name="url" column="url" type="VARCHAR"/>
+ <field name="content" column="content" type="BINARY"/>
+ <field name="parsedContent" column="parsedContent" type="BINARY"/>
+ <field name="outlinks" column="outlinks" type="BINARY"/>
+ <field name="headers" column="headers" type="BINARY"/>
+ <field name="metadata" column="metadata" type="BINARY"/>
+ <field name="byteData" column="byteData" type="BINARY"/>
+ <field name="stringData" column="stringData" type="VARCHAR"/>
+ </class>
+</gora-otd>
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/test/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora.properties b/gora-ignite/src/test/resources/gora.properties
index 5cf0e25..e45aab7 100644
--- a/gora-ignite/src/test/resources/gora.properties
+++ b/gora-ignite/src/test/resources/gora.properties
@@ -14,8 +14,9 @@
# limitations under the License.
gora.datastore.default=org.apache.gora.ignite.store.IgniteStore
-gora.datastore.ignite.mock=false
-gora.datastore.ignite.instance=goraTest
-gora.datastore.ignite.zookeepers=localhost:56321
-gora.datastore.ignite.user=root
-gora.datastore.ignite.password=drowssap
\ No newline at end of file
+gora.datastore.ignite.schema=PUBLIC
+gora.datastore.ignite.host=localhost
+gora.datastore.ignite.port=10800
+gora.datastore.ignite.user=
+gora.datastore.ignite.password=
+gora.datastore.ignite.additionalConfigurations=
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/nbactions.xml
----------------------------------------------------------------------
diff --git a/nbactions.xml b/nbactions.xml
new file mode 100644
index 0000000..52ecdda
--- /dev/null
+++ b/nbactions.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+ <action>
+ <actionName>rebuild</actionName>
+ <packagings>
+ <packaging>*</packaging>
+ </packagings>
+ <goals>
+ <goal>clean</goal>
+ <goal>install</goal>
+ </goals>
+ </action>
+ </actions>
[06/11] gora git commit: Implement schema handling methods
Posted by dj...@apache.org.
Implement schema handling methods
The basic schema creation/deletion methods are implemented for the
Ignite backend. In addition, all the SQL generation code was moved to
a separate class.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/b1e2ae4c
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/b1e2ae4c
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/b1e2ae4c
Branch: refs/heads/master
Commit: b1e2ae4c902691bbce02a90b76c4bee1c6e58455
Parents: 7545170
Author: Carlos M <ca...@gmail.com>
Authored: Tue Jun 26 23:18:19 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Tue Jun 26 23:18:19 2018 -0500
----------------------------------------------------------------------
.../apache/gora/ignite/store/IgniteStore.java | 37 +++++++++--
.../gora/ignite/utils/IgniteSQLBuilder.java | 69 ++++++++++++++++++++
gora-ignite/src/test/resources/gora.properties | 4 +-
3 files changed, 101 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/b1e2ae4c/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
index d52d10e..ebc2943 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -22,11 +22,9 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
-import java.text.MessageFormat;
import java.util.List;
-import java.util.Locale;
import java.util.Properties;
-import java.util.logging.Level;
+import org.apache.gora.ignite.utils.IgniteSQLBuilder;
import org.apache.gora.persistency.impl.PersistentBase;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
@@ -105,20 +103,45 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public void createSchema() throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ if (connection == null) {
+ throw new GoraException(
+ "Impossible to create the schema as no connection has been initiated.");
+ }
+ if (schemaExists()) {
+ return;
+ }
+ try (Statement stmt = connection.createStatement()) {
+ String createTableSQL = IgniteSQLBuilder.createTable(igniteMapping);
+ stmt.executeUpdate(createTableSQL);
+ LOG.info("Table {} has been created for Ignite instance.",
+ igniteMapping.getTableName());
+ } catch (SQLException ex) {
+ throw new GoraException(ex);
+ }
}
@Override
public void deleteSchema() throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ if (connection == null) {
+ throw new GoraException(
+ "Impossible to delete the schema as no connection has been initiated.");
+ }
+ try (Statement stmt = connection.createStatement()) {
+ String dropTableSQL = IgniteSQLBuilder.dropTable(igniteMapping.getTableName());
+ stmt.executeUpdate(dropTableSQL);
+ LOG.info("Table {} has been dropped from Ignite instance.",
+ igniteMapping.getTableName());
+ } catch (SQLException ex) {
+ throw new GoraException(ex);
+ }
}
@Override
public boolean schemaExists() throws GoraException {
boolean exists = false;
try (Statement stmt = connection.createStatement()) {
- MessageFormat messageFormat = new MessageFormat("select * from {0} limit 0", Locale.getDefault());
- ResultSet executeQuery = stmt.executeQuery(messageFormat.format(igniteMapping.getTableName()));
+ String tableExistsSQL = IgniteSQLBuilder.tableExists(igniteMapping.getTableName());
+ ResultSet executeQuery = stmt.executeQuery(tableExistsSQL);
executeQuery.close();
exists = true;
} catch (SQLException ex) {
http://git-wip-us.apache.org/repos/asf/gora/blob/b1e2ae4c/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
new file mode 100644
index 0000000..0480fd4
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.utils;
+
+import avro.shaded.com.google.common.collect.Lists;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Locale;
+import java.util.Map;
+import org.apache.gora.ignite.store.Column;
+import org.apache.gora.ignite.store.IgniteMapping;
+
+/**
+ *
+ * SQL Builder utility for Ignite.
+ */
+public class IgniteSQLBuilder {
+
+ private static String format(String pattern, Object... args) {
+ MessageFormat messageFormat = new MessageFormat(pattern, Locale.getDefault());
+ return messageFormat.format(args);
+ }
+
+ public static String tableExists(String tableName) {
+ return format("SELECT * FROM {0} LIMIT 0", tableName);
+ }
+
+ public static String createTable(IgniteMapping mapping) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("CREATE TABLE ");
+ sqlBuilder.append(mapping.getTableName());
+ sqlBuilder.append("(");
+ ArrayList<Map.Entry<String, Column>> fieldsList = Lists.newArrayList(mapping.getFields().entrySet());
+ for (Map.Entry<String, Column> aField : fieldsList) {
+ Column aColumn = aField.getValue();
+ String name = aColumn.getName();
+ Column.FieldType dataType = aColumn.getDataType();
+ sqlBuilder.append(name).append(" ").append(dataType.toString()).append(",");
+ }
+ sqlBuilder.append("PRIMARY KEY ");
+ sqlBuilder.append("(");
+ for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
+ sqlBuilder.append(mapping.getPrimaryKey().get(i));
+ sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : ",");
+ }
+ sqlBuilder.append(")");
+ sqlBuilder.append(");");
+ return sqlBuilder.toString();
+ }
+
+ public static String dropTable(String tableName) {
+ return format("DROP TABLE IF EXISTS {0} ;", tableName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/b1e2ae4c/gora-ignite/src/test/resources/gora.properties
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora.properties b/gora-ignite/src/test/resources/gora.properties
index e45aab7..e0d1901 100644
--- a/gora-ignite/src/test/resources/gora.properties
+++ b/gora-ignite/src/test/resources/gora.properties
@@ -17,6 +17,6 @@ gora.datastore.default=org.apache.gora.ignite.store.IgniteStore
gora.datastore.ignite.schema=PUBLIC
gora.datastore.ignite.host=localhost
gora.datastore.ignite.port=10800
-gora.datastore.ignite.user=
-gora.datastore.ignite.password=
+#gora.datastore.ignite.user=
+#gora.datastore.ignite.password=
gora.datastore.ignite.additionalConfigurations=
\ No newline at end of file
[07/11] gora git commit: Implement put, get, delete
Posted by dj...@apache.org.
Implement put, get, delete
The operations put, get and delete are implemented for the
Ignite backend. In addition, primary key mapping is
improved.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/db7540e0
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/db7540e0
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/db7540e0
Branch: refs/heads/master
Commit: db7540e045fc07ef39d37163a263615c6e580efe
Parents: b1e2ae4
Author: Carlos M <ca...@gmail.com>
Authored: Mon Jul 2 00:50:43 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Mon Jul 2 00:50:43 2018 -0500
----------------------------------------------------------------------
.../apache/gora/ignite/store/IgniteMapping.java | 6 +-
.../gora/ignite/store/IgniteMappingBuilder.java | 6 +-
.../apache/gora/ignite/store/IgniteStore.java | 323 ++++++++++++++++++-
.../gora/ignite/utils/IgniteSQLBuilder.java | 90 +++++-
.../src/test/resources/gora-ignite-mapping.xml | 10 +-
5 files changed, 412 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/db7540e0/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
index 2e59da1..c0e7a98 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
@@ -27,7 +27,7 @@ public class IgniteMapping {
private String tableName;
private Map<String, Column> fields;
- private List<String> primaryKey;
+ private List<Column> primaryKey;
public IgniteMapping() {
fields = new HashMap<>();
@@ -49,11 +49,11 @@ public class IgniteMapping {
this.fields = fields;
}
- public List<String> getPrimaryKey() {
+ public List<Column> getPrimaryKey() {
return primaryKey;
}
- public void setPrimaryKey(List<String> primaryKey) {
+ public void setPrimaryKey(List<Column> primaryKey) {
this.primaryKey = primaryKey;
}
http://git-wip-us.apache.org/repos/asf/gora/blob/db7540e0/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
index 81a3fae..3d245ea 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
@@ -80,9 +80,11 @@ public class IgniteMappingBuilder<K, T extends PersistentBase> {
String tableName = dataStore.getSchemaName(tableNameFromMapping, dataStore.getPersistentClass());
igniteMapping.setTableName(tableName);
List<Element> prColumns = classElement.getChildren("primarykey");
- List<String> prFields = new ArrayList<>();
+ List<Column> prFields = new ArrayList<>();
for (Element aPrimaryKey : prColumns) {
- prFields.add(aPrimaryKey.getAttributeValue("column"));
+ String name = aPrimaryKey.getAttributeValue("column");
+ String type = aPrimaryKey.getAttributeValue("type");
+ prFields.add(new Column(name, Column.FieldType.valueOf(type)));
}
igniteMapping.setPrimaryKey(prFields);
List<Element> fields = classElement.getChildren("field");
http://git-wip-us.apache.org/repos/asf/gora/blob/db7540e0/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
index ebc2943..497e161 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -17,20 +17,34 @@
package org.apache.gora.ignite.store;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.avro.Schema;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.util.Utf8;
import org.apache.gora.ignite.utils.IgniteSQLBuilder;
+import org.apache.gora.persistency.Persistent;
import org.apache.gora.persistency.impl.PersistentBase;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
import org.apache.gora.store.impl.DataStoreBase;
+import org.apache.gora.util.AvroUtils;
import org.apache.gora.util.GoraException;
+import org.apache.gora.util.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,6 +63,16 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
private IgniteMapping igniteMapping;
private Connection connection;
+ /*
+ * Create a threadlocal map for the datum readers and writers, because they
+ * are not thread safe, at least not before Avro 1.4.0 (See AVRO-650). When
+ * they are thread safe, it is possible to maintain a single reader and writer
+ * pair for every schema, instead of one for every thread.
+ */
+ public static final ConcurrentHashMap<Schema, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<>();
+
+ public static final ConcurrentHashMap<Schema, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<>();
+
@Override
public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) throws GoraException {
@@ -161,17 +185,151 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public T get(K key, String[] fields) throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ String[] avFields = getFieldsToQuery(fields);
+ Object[] keyl = null;
+ if (igniteMapping.getPrimaryKey().size() == 1) {
+ keyl = new Object[]{key};
+ } else {
+ //Composite key pending
+ }
+ //Avro fields to Ignite fields
+ List<String> dbFields = new ArrayList<>();
+ for (String af : avFields) {
+ dbFields.add(igniteMapping.getFields().get(af).getName());
+ }
+ String selectQuery = IgniteSQLBuilder.selectGet(igniteMapping, dbFields);
+ try (PreparedStatement stmt = connection.prepareStatement(selectQuery)) {
+ IgniteSQLBuilder.fillSelectStatement(stmt, igniteMapping, keyl);
+ ResultSet rs = stmt.executeQuery();
+ boolean data = rs.next();
+ T resp = null;
+ if (data) {
+ resp = newInstance(rs, fields);
+ if (rs.next()) {
+ LOG.warn("Multiple results for primary key {} in the schema {}, ignoring additional rows.", keyl, igniteMapping.getTableName());
+ }
+ }
+ rs.close();
+ return resp;
+ } catch (SQLException | IOException ex) {
+ throw new GoraException(ex);
+ }
+
+ }
+
+ public T newInstance(ResultSet rs, String[] fields) throws GoraException, SQLException, IOException {
+ fields = getFieldsToQuery(fields);
+ T persistent = newPersistent();
+ for (String f : fields) {
+ Schema.Field field = fieldMap.get(f);
+ Schema fieldSchema = field.schema();
+ String dbField = igniteMapping.getFields().get(f).getName();
+ Object sv = rs.getObject(dbField);
+ if (sv == null) {
+ continue;
+ }
+ Object v = deserializeFieldValue(field, fieldSchema, sv, persistent);
+ persistent.put(field.pos(), v);
+ persistent.setDirty(field.pos());
+ }
+ return persistent;
+ }
+
+ private Object deserializeFieldValue(Schema.Field field, Schema fieldSchema,
+ Object igniteValue, T persistent) throws IOException {
+ Object fieldValue = null;
+ switch (fieldSchema.getType()) {
+ case MAP:
+ case ARRAY:
+ case RECORD:
+ @SuppressWarnings("rawtypes") SpecificDatumReader reader = getDatumReader(fieldSchema);
+ fieldValue = IOUtils.deserialize((byte[]) igniteValue, reader,
+ persistent.get(field.pos()));
+ break;
+ case ENUM:
+ fieldValue = AvroUtils.getEnumValue(fieldSchema, igniteValue.toString());
+ break;
+ case FIXED:
+ break;
+ case BYTES:
+ fieldValue = ByteBuffer.wrap((byte[]) igniteValue);
+ break;
+ case STRING:
+ fieldValue = new Utf8(igniteValue.toString());
+ break;
+ case UNION:
+ if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) {
+ int schemaPos = getUnionSchema(igniteValue, fieldSchema);
+ Schema unionSchema = fieldSchema.getTypes().get(schemaPos);
+ fieldValue = deserializeFieldValue(field, unionSchema, igniteValue, persistent);
+ } else {
+ reader = getDatumReader(fieldSchema);
+ fieldValue = IOUtils.deserialize((byte[]) igniteValue, reader,
+ persistent.get(field.pos()));
+ }
+ break;
+ default:
+ fieldValue = igniteValue;
+ }
+ return fieldValue;
+
}
@Override
public void put(K key, T obj) throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ try {
+ if (obj.isDirty()) {
+ Schema schema = obj.getSchema();
+ List<Schema.Field> fields = schema.getFields();
+ Map<Column, Object> data = new HashMap<>();
+ if (igniteMapping.getPrimaryKey().size() == 1) {
+ Column getKey = igniteMapping.getPrimaryKey().get(0);
+ data.put(getKey, key);
+ } else {
+ //Composite keys pending..
+ }
+ for (Schema.Field field : fields) {
+ Column get = igniteMapping.getFields().get(field.name());
+ Object v = obj.get(field.pos());
+ if (get != null && v != null) {
+ Schema fieldSchema = field.schema();
+ Object serializedObj = serializeFieldValue(get, fieldSchema, v);
+ data.put(get, serializedObj);
+ }
+ }
+ String baseInsertStatement = IgniteSQLBuilder.baseInsertStatement(igniteMapping, data);
+ try (PreparedStatement stmt = connection.prepareStatement(baseInsertStatement)) {
+ IgniteSQLBuilder.fillInsertStatement(stmt, data);
+ stmt.executeUpdate();
+ } catch (SQLException ex) {
+ throw new GoraException(ex);
+ }
+ } else {
+ LOG.info("Ignored putting object {} in the store as it is neither "
+ + "new, neither dirty.", new Object[]{obj});
+ }
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
}
@Override
public boolean delete(K key) throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ String deleteQuery = null;
+ Object[] keyl = null;
+ if (igniteMapping.getPrimaryKey().size() == 1) {
+ deleteQuery = IgniteSQLBuilder.delete(igniteMapping);
+ keyl = new Object[]{key};
+ } else {
+ //Composite key pending
+ }
+ try (PreparedStatement stmt = connection.prepareStatement(deleteQuery)) {
+ IgniteSQLBuilder.fillDeleteStatement(stmt, igniteMapping, keyl);
+ stmt.executeUpdate();
+ return true;
+ } catch (SQLException ex) {
+ throw new GoraException(ex);
+ }
}
@Override
@@ -196,11 +354,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public void flush() throws GoraException {
- try {
- connection.commit();
- } catch (Exception e) {
- throw new GoraException(e);
- }
+ //Auto-commit mode by default
}
@Override
@@ -213,4 +367,157 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
}
}
+ private Object serializeFieldValue(Column get, Schema fieldSchema, Object fieldValue) {
+ Object output = fieldValue;
+ switch (fieldSchema.getType()) {
+ case ARRAY:
+ case MAP:
+ case RECORD:
+ byte[] data = null;
+ try {
+ @SuppressWarnings("rawtypes")
+ SpecificDatumWriter writer = getDatumWriter(fieldSchema);
+ data = IOUtils.serialize(writer, fieldValue);
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ output = data;
+ break;
+ case UNION:
+ if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) {
+ int schemaPos = getUnionSchema(fieldValue, fieldSchema);
+ Schema unionSchema = fieldSchema.getTypes().get(schemaPos);
+ output = serializeFieldValue(get, unionSchema, fieldValue);
+ } else {
+ data = null;
+ try {
+ @SuppressWarnings("rawtypes")
+ SpecificDatumWriter writer = getDatumWriter(fieldSchema);
+ data = IOUtils.serialize(writer, fieldValue);
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ output = data;
+ }
+ break;
+ case FIXED:
+ break;
+ case ENUM:
+ case STRING:
+ output = fieldValue.toString();
+ break;
+ case BYTES:
+ output = ((ByteBuffer) fieldValue).array();
+ break;
+ case INT:
+ case LONG:
+ case FLOAT:
+ case DOUBLE:
+ case BOOLEAN:
+ output = fieldValue;
+ break;
+ case NULL:
+ break;
+ default:
+ throw new AssertionError(fieldSchema.getType().name());
+
+ }
+ return output;
+ }
+
+ private boolean isNullable(Schema unionSchema) {
+ for (Schema innerSchema : unionSchema.getTypes()) {
+ if (innerSchema.getType().equals(Schema.Type.NULL)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Method to retrieve the corresponding schema type index of a particular
+ * object having UNION schema. As UNION type can have one or more types and at
+ * a given instance, it holds an object of only one type of the defined types,
+ * this method is used to figure out the corresponding instance's schema type
+ * index.
+ *
+ * @param instanceValue value that the object holds
+ * @param unionSchema union schema containing all of the data types
+ * @return the unionSchemaPosition corresponding schema position
+ */
+ private int getUnionSchema(Object instanceValue, Schema unionSchema) {
+ int unionSchemaPos = 0;
+ for (Schema currentSchema : unionSchema.getTypes()) {
+ Schema.Type schemaType = currentSchema.getType();
+ if (instanceValue instanceof CharSequence && schemaType.equals(Schema.Type.STRING)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof ByteBuffer && schemaType.equals(Schema.Type.BYTES)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof byte[] && schemaType.equals(Schema.Type.BYTES)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Integer && schemaType.equals(Schema.Type.INT)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Long && schemaType.equals(Schema.Type.LONG)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Double && schemaType.equals(Schema.Type.DOUBLE)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Float && schemaType.equals(Schema.Type.FLOAT)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Boolean && schemaType.equals(Schema.Type.BOOLEAN)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Map && schemaType.equals(Schema.Type.MAP)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof List && schemaType.equals(Schema.Type.ARRAY)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof Persistent && schemaType.equals(Schema.Type.RECORD)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof byte[] && schemaType.equals(Schema.Type.MAP)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof byte[] && schemaType.equals(Schema.Type.RECORD)) {
+ return unionSchemaPos;
+ }
+ if (instanceValue instanceof byte[] && schemaType.equals(Schema.Type.ARRAY)) {
+ return unionSchemaPos;
+ }
+ unionSchemaPos++;
+ }
+ return 0;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private SpecificDatumReader getDatumReader(Schema fieldSchema) {
+ SpecificDatumReader<?> reader = readerMap.get(fieldSchema);
+ if (reader == null) {
+ reader = new SpecificDatumReader(fieldSchema);// ignore dirty bits
+ SpecificDatumReader localReader = null;
+ if ((localReader = readerMap.putIfAbsent(fieldSchema, reader)) != null) {
+ reader = localReader;
+ }
+ }
+ return reader;
+ }
+
+ @SuppressWarnings("rawtypes")
+ private SpecificDatumWriter getDatumWriter(Schema fieldSchema) {
+ SpecificDatumWriter writer = writerMap.get(fieldSchema);
+ if (writer == null) {
+ writer = new SpecificDatumWriter(fieldSchema);// ignore dirty bits
+ writerMap.put(fieldSchema, writer);
+ }
+
+ return writer;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/db7540e0/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
index 0480fd4..92014c1 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
@@ -17,10 +17,15 @@
package org.apache.gora.ignite.utils;
import avro.shaded.com.google.common.collect.Lists;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
import java.text.MessageFormat;
import java.util.ArrayList;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import org.apache.gora.ignite.store.Column;
import org.apache.gora.ignite.store.IgniteMapping;
@@ -44,9 +49,9 @@ public class IgniteSQLBuilder {
sqlBuilder.append("CREATE TABLE ");
sqlBuilder.append(mapping.getTableName());
sqlBuilder.append("(");
- ArrayList<Map.Entry<String, Column>> fieldsList = Lists.newArrayList(mapping.getFields().entrySet());
- for (Map.Entry<String, Column> aField : fieldsList) {
- Column aColumn = aField.getValue();
+ ArrayList<Column> fieldsList = Lists.newArrayList(mapping.getPrimaryKey());
+ fieldsList.addAll(Lists.newArrayList(mapping.getFields().values()));
+ for (Column aColumn : fieldsList) {
String name = aColumn.getName();
Column.FieldType dataType = aColumn.getDataType();
sqlBuilder.append(name).append(" ").append(dataType.toString()).append(",");
@@ -54,16 +59,91 @@ public class IgniteSQLBuilder {
sqlBuilder.append("PRIMARY KEY ");
sqlBuilder.append("(");
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
- sqlBuilder.append(mapping.getPrimaryKey().get(i));
+ sqlBuilder.append(mapping.getPrimaryKey().get(i).getName());
sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : ",");
}
sqlBuilder.append(")");
sqlBuilder.append(");");
return sqlBuilder.toString();
}
-
+
public static String dropTable(String tableName) {
return format("DROP TABLE IF EXISTS {0} ;", tableName);
}
+ public static String baseInsertStatement(IgniteMapping mapping, Map<Column, Object> data) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("MERGE INTO ");
+ sqlBuilder.append(mapping.getTableName());
+ sqlBuilder.append(" (");
+ List<Entry<Column, Object>> list = new ArrayList<>(data.entrySet());
+ for (int i = 0; i < list.size(); i++) {
+ sqlBuilder.append(list.get(i).getKey().getName());
+ sqlBuilder.append(i == list.size() - 1 ? "" : ",");
+ }
+ sqlBuilder.append(")");
+ sqlBuilder.append(" VALUES ");
+ sqlBuilder.append(" (");
+ for (int i = 0; i < list.size(); i++) {
+ sqlBuilder.append("?");
+ sqlBuilder.append(i == list.size() - 1 ? "" : ",");
+ }
+ sqlBuilder.append(" )");
+ return sqlBuilder.toString();
+ }
+
+ public static void fillInsertStatement(PreparedStatement st, Map<Column, Object> data) throws SQLException {
+ List<Entry<Column, Object>> list = new ArrayList<>(data.entrySet());
+ for (int i = 0; i < list.size(); i++) {
+ int j = i + 1;
+ st.setObject(j, list.get(i).getValue());
+ }
+ }
+
+ public static String delete(IgniteMapping mapping) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("DELETE FROM ");
+ sqlBuilder.append(mapping.getTableName());
+ sqlBuilder.append(" WHERE ");
+ for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
+ sqlBuilder.append(mapping.getPrimaryKey().get(i).getName());
+ sqlBuilder.append("= ? ");
+ sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : " AND ");
+ }
+ return sqlBuilder.toString();
+ }
+
+ public static void fillDeleteStatement(PreparedStatement st, IgniteMapping mapping, Object... data) throws SQLException {
+ assert mapping.getPrimaryKey().size() == data.length;
+ for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
+ int j = i + 1;
+ st.setObject(j, data[i]);
+ }
+ }
+
+ public static String selectGet(IgniteMapping mapping, List<String> fields) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("SELECT ");
+ for (int i = 0; i < fields.size(); i++) {
+ sqlBuilder.append(fields.get(i));
+ sqlBuilder.append(i == fields.size() - 1 ? "" : " , ");
+ }
+ sqlBuilder.append(" FROM ");
+ sqlBuilder.append(mapping.getTableName());
+ sqlBuilder.append(" WHERE ");
+ for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
+ sqlBuilder.append(mapping.getPrimaryKey().get(i).getName());
+ sqlBuilder.append("= ? ");
+ sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : " AND ");
+ }
+ return sqlBuilder.toString();
+ }
+
+ public static void fillSelectStatement(PreparedStatement st, IgniteMapping mapping, Object... data) throws SQLException {
+ assert mapping.getPrimaryKey().size() == data.length;
+ for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
+ int j = i + 1;
+ st.setObject(j, data[i]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/gora/blob/db7540e0/gora-ignite/src/test/resources/gora-ignite-mapping.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/gora-ignite-mapping.xml b/gora-ignite/src/test/resources/gora-ignite-mapping.xml
index 3c13774..6368e18 100644
--- a/gora-ignite/src/test/resources/gora-ignite-mapping.xml
+++ b/gora-ignite/src/test/resources/gora-ignite-mapping.xml
@@ -19,17 +19,17 @@
<gora-otd>
<class name="org.apache.gora.examples.generated.Employee" keyClass="java.lang.String" table="Employee">
- <primarykey column="ssn"/>
+ <primarykey column="pkssn" type="VARCHAR" />
<field name="ssn" column="ssn" type="VARCHAR"/>
<field name="name" column="name" type="VARCHAR"/>
<field name="dateOfBirth" column="dateOfBirth" type="BIGINT"/>
<field name="salary" column="salary" type="INT"/>
- <field name="boss" column="boss" type="VARCHAR"/>
- <field name="webpage" column="webpage" type="VARCHAR"/>
+ <field name="boss" column="boss" type="BINARY"/>
+ <field name="webpage" column="webpage" type="BINARY"/>
</class>
<class name="org.apache.gora.examples.generated.WebPage" keyClass="java.lang.String" table="WebPage">
- <primarykey column="url"/>
+ <primarykey column="pkurl" type="VARCHAR"/>
<field name="url" column="url" type="VARCHAR"/>
<field name="content" column="content" type="BINARY"/>
<field name="parsedContent" column="parsedContent" type="BINARY"/>
@@ -37,6 +37,6 @@
<field name="headers" column="headers" type="BINARY"/>
<field name="metadata" column="metadata" type="BINARY"/>
<field name="byteData" column="byteData" type="BINARY"/>
- <field name="stringData" column="stringData" type="VARCHAR"/>
+ <field name="stringData" column="stringData" type="BINARY"/>
</class>
</gora-otd>
[02/11] gora git commit: create template gora-ignite module
Posted by dj...@apache.org.
create template gora-ignite module
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/4346216f
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/4346216f
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/4346216f
Branch: refs/heads/master
Commit: 4346216ffbed9a9adef2b90589436de543167c4c
Parents: fb4b355
Author: Carlos Rodrigo Muñoz <ca...@gmail.com>
Authored: Mon May 7 23:00:44 2018 -0500
Committer: Carlos Rodrigo Muñoz <ca...@gmail.com>
Committed: Mon May 7 23:01:18 2018 -0500
----------------------------------------------------------------------
gora-ignite/pom.xml | 177 +++
gora-ignite/src/examples/java/.gitignore | 15 +
.../gora/ignite/encoders/BinaryEncoder.java | 191 ++++
.../apache/gora/ignite/encoders/Encoder.java | 72 ++
.../apache/gora/ignite/encoders/HexEncoder.java | 204 ++++
.../ignite/encoders/SignedBinaryEncoder.java | 110 ++
.../org/apache/gora/ignite/encoders/Utils.java | 91 ++
.../gora/ignite/encoders/package-info.java | 20 +
.../org/apache/gora/ignite/package-info.java | 20 +
.../apache/gora/ignite/query/IgniteQuery.java | 45 +
.../apache/gora/ignite/query/IgniteResult.java | 101 ++
.../apache/gora/ignite/query/package-info.java | 21 +
.../apache/gora/ignite/store/IgniteMapping.java | 44 +
.../apache/gora/ignite/store/IgniteStore.java | 1034 ++++++++++++++++++
.../apache/gora/ignite/store/package-info.java | 20 +
.../ignite/util/FixedByteArrayOutputStream.java | 45 +
.../apache/gora/ignite/util/package-info.java | 20 +
.../gora/ignite/GoraIgniteTestDriver.java | 73 ++
.../org/apache/gora/ignite/package-info.java | 21 +
.../ignite/store/AuthenticationTokenTest.java | 90 ++
.../gora/ignite/store/IgniteStoreTest.java | 87 ++
.../apache/gora/ignite/store/PartitionTest.java | 96 ++
.../apache/gora/ignite/store/package-info.java | 21 +
.../apache/gora/ignite/util/HexEncoderTest.java | 56 +
.../ignite/util/SignedBinaryEncoderTest.java | 167 +++
.../apache/gora/ignite/util/package-info.java | 20 +
.../test/resources/gora-accumulo-mapping.xml | 59 +
gora-ignite/src/test/resources/gora.properties | 21 +
pom.xml | 1 +
29 files changed, 2942 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/pom.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/pom.xml b/gora-ignite/pom.xml
new file mode 100644
index 0000000..ddd3a4a
--- /dev/null
+++ b/gora-ignite/pom.xml
@@ -0,0 +1,177 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora</artifactId>
+ <version>0.9-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
+ <artifactId>gora-ignite</artifactId>
+ <packaging>bundle</packaging>
+
+ <name>Apache Gora :: Ignite</name>
+ <url>http://gora.apache.org</url>
+ <description>The Apache Gora open source framework provides an in-memory data model and
+ persistence for big data. Gora supports persisting to column stores, key value stores,
+ document stores and RDBMSs, and analyzing the data with extensive Apache Hadoop MapReduce
+ support.</description>
+ <inceptionYear>2010</inceptionYear>
+ <organization>
+ <name>The Apache Software Foundation</name>
+ <url>http://www.apache.org/</url>
+ </organization>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/GORA</url>
+ </issueManagement>
+ <ciManagement>
+ <system>Jenkins</system>
+ <url>https://builds.apache.org/job/Gora-trunk/</url>
+ </ciManagement>
+
+ <properties>
+ <ignite.version>1.7.1</ignite.version>
+ <osgi.import>*</osgi.import>
+ <osgi.export>org.apache.gora.ignite*;version="${project.version}";-noimport:=true</osgi.export>
+ </properties>
+
+ <build>
+ <directory>target</directory>
+ <outputDirectory>target/classes</outputDirectory>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <testOutputDirectory>target/test-classes</testOutputDirectory>
+ <testSourceDirectory>src/test/java</testSourceDirectory>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <testResources>
+ <testResource>
+ <directory>${project.basedir}/src/test/resources</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ <!--targetPath>${project.basedir}/target/classes/</targetPath -->
+ </testResource>
+ </testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>${build-helper-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/examples/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <!-- Gora Internal Dependencies -->
+ <dependency>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <!--Ignite Dependency -->
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${ignite.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-minicluster</artifactId>
+ <version>${ignite.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ <dependency>
+ <!-- Ignite mini was trying to load the Shell, which failed with the version of jline in the parent pom -->
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ <version>2.11</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+
+ <!-- Logging Dependencies -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>javax.jms</groupId>
+ <artifactId>jms</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <!-- Testing Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ </dependency>
+
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/examples/java/.gitignore
----------------------------------------------------------------------
diff --git a/gora-ignite/src/examples/java/.gitignore b/gora-ignite/src/examples/java/.gitignore
new file mode 100644
index 0000000..09697dc
--- /dev/null
+++ b/gora-ignite/src/examples/java/.gitignore
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java
new file mode 100644
index 0000000..ce6eded
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.encoders;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.gora.ignite.util.FixedByteArrayOutputStream;
+
+/**
+ *
+ */
+public class BinaryEncoder implements Encoder {
+
+ @Override
+ public byte[] encodeShort(short s) throws IOException {
+ return encodeShort(s, new byte[2]);
+ }
+
+ @Override
+ public byte[] encodeShort(short s, byte[] ret) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
+ dos.writeShort(s);
+ dos.close();
+ return ret;
+ }
+ }
+
+ @Override
+ public short decodeShort(byte[] a) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
+ short s = dis.readShort();
+ dis.close();
+ return s;
+ }
+ }
+
+ @Override
+ public byte[] encodeInt(int i) throws IOException {
+ return encodeInt(i, new byte[4]);
+ }
+
+ @Override
+ public byte[] encodeInt(int i, byte[] ret) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
+ dos.writeInt(i);
+ dos.close();
+ return ret;
+ }
+ }
+
+ @Override
+ public int decodeInt(byte[] a) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
+ int i = dis.readInt();
+ dis.close();
+ return i;
+ }
+ }
+
+ @Override
+ public byte[] encodeLong(long l) throws IOException {
+ return encodeLong(l, new byte[8]);
+ }
+
+ @Override
+ public byte[] encodeLong(long l, byte[] ret) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
+ dos.writeLong(l);
+ dos.close();
+ return ret;
+ }
+ }
+
+ @Override
+ public long decodeLong(byte[] a) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
+ long l = dis.readLong();
+ dis.close();
+ return l;
+ }
+ }
+
+ @Override
+ public byte[] encodeDouble(double d) throws IOException {
+ return encodeDouble(d, new byte[8]);
+ }
+
+ @Override
+ public byte[] encodeDouble(double d, byte[] ret) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
+ long l = Double.doubleToRawLongBits(d);
+ dos.writeLong(l);
+ dos.close();
+ return ret;
+ }
+ }
+
+ @Override
+ public double decodeDouble(byte[] a) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
+ long l = dis.readLong();
+ dis.close();
+ return Double.longBitsToDouble(l);
+ }
+ }
+
+ @Override
+ public byte[] encodeFloat(float d) throws IOException {
+ return encodeFloat(d, new byte[4]);
+ }
+
+ @Override
+ public byte[] encodeFloat(float f, byte[] ret) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
+ int i = Float.floatToRawIntBits(f);
+ dos.writeInt(i);
+ return ret;
+ }
+ }
+
+ @Override
+ public float decodeFloat(byte[] a) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
+ int i = dis.readInt();
+ return Float.intBitsToFloat(i);
+ }
+ }
+
+ @Override
+ public byte[] encodeByte(byte b, byte[] ret) {
+ ret[0] = 0;
+ return ret;
+ }
+
+ @Override
+ public byte[] encodeByte(byte b) {
+ return encodeByte(b, new byte[1]);
+ }
+
+ @Override
+ public byte decodeByte(byte[] a) {
+ return a[0];
+ }
+
+ @Override
+ public boolean decodeBoolean(byte[] a) throws IOException {
+ try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
+ return dis.readBoolean();
+ }
+ }
+
+ @Override
+ public byte[] encodeBoolean(boolean b) throws IOException {
+ return encodeBoolean(b, new byte[1]);
+ }
+
+ @Override
+ public byte[] encodeBoolean(boolean b, byte[] ret) throws IOException {
+ try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
+ dos.writeBoolean(b);
+ return ret;
+ }
+ }
+
+ @Override
+ public byte[] lastPossibleKey(int size, byte[] er) {
+ return Utils.lastPossibleKey(size, er);
+ }
+
+ @Override
+ public byte[] followingKey(int size, byte[] per) {
+ return Utils.followingKey(size, per);
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java
new file mode 100644
index 0000000..deea4a7
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.encoders;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public interface Encoder {
+
+ public byte[] encodeByte(byte b, byte[] ret);
+
+ public byte[] encodeByte(byte b);
+
+ public byte decodeByte(byte[] a);
+
+ public byte[] encodeShort(short s) throws IOException;
+
+ public byte[] encodeShort(short s, byte[] ret) throws IOException;
+
+ public short decodeShort(byte[] a) throws IOException;
+
+ public byte[] encodeInt(int i) throws IOException;
+
+ public byte[] encodeInt(int i, byte[] ret) throws IOException;
+
+ public int decodeInt(byte[] a) throws IOException;
+
+ public byte[] encodeLong(long l) throws IOException;
+
+ public byte[] encodeLong(long l, byte[] ret) throws IOException;
+
+ public long decodeLong(byte[] a) throws IOException;
+
+ public byte[] encodeDouble(double d) throws IOException;
+
+ public byte[] encodeDouble(double d, byte[] ret) throws IOException;
+
+ public double decodeDouble(byte[] a) throws IOException;
+
+ public byte[] encodeFloat(float d) throws IOException;
+
+ public byte[] encodeFloat(float f, byte[] ret) throws IOException;
+
+ public float decodeFloat(byte[] a) throws IOException;
+
+ public boolean decodeBoolean(byte[] val) throws IOException;
+
+ public byte[] encodeBoolean(boolean b) throws IOException;
+
+ public byte[] encodeBoolean(boolean b, byte[] ret) throws IOException;
+
+ byte[] followingKey(int size, byte[] per);
+
+ byte[] lastPossibleKey(int size, byte[] er);
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java
new file mode 100644
index 0000000..8568ba9
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.encoders;
+
+/**
+ * Encodes data in a ascii hex representation
+ */
+
+public class HexEncoder implements Encoder {
+
+ private byte[] chars = new byte[] {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
+
+ private void encode(byte[] a, long l) {
+ for (int i = a.length - 1; i >= 0; i--) {
+ a[i] = chars[(int) (l & 0x0f)];
+ l = l >>> 4;
+ }
+ }
+
+ private int fromChar(byte b) {
+ if (b >= '0' && b <= '9') {
+ return b - '0';
+ } else if (b >= 'a' && b <= 'f') {
+ return b - 'a' + 10;
+ }
+
+ throw new IllegalArgumentException("Bad char " + b);
+ }
+
+ private long decode(byte[] a) {
+ long b = 0;
+ for (byte anA : a) {
+ b = b << 4;
+ b |= fromChar(anA);
+ }
+
+ return b;
+ }
+
+ @Override
+ public byte[] encodeByte(byte b, byte[] ret) {
+ encode(ret, 0xff & b);
+ return ret;
+ }
+
+ @Override
+ public byte[] encodeByte(byte b) {
+ return encodeByte(b, new byte[2]);
+ }
+
+ @Override
+ public byte decodeByte(byte[] a) {
+ return (byte) decode(a);
+ }
+
+ @Override
+ public byte[] encodeShort(short s) {
+ return encodeShort(s, new byte[4]);
+ }
+
+ @Override
+ public byte[] encodeShort(short s, byte[] ret) {
+ encode(ret, 0xffff & s);
+ return ret;
+ }
+
+ @Override
+ public short decodeShort(byte[] a) {
+ return (short) decode(a);
+ }
+
+ @Override
+ public byte[] encodeInt(int i) {
+ return encodeInt(i, new byte[8]);
+ }
+
+ @Override
+ public byte[] encodeInt(int i, byte[] ret) {
+ encode(ret, i);
+ return ret;
+ }
+
+ @Override
+ public int decodeInt(byte[] a) {
+ return (int) decode(a);
+ }
+
+ @Override
+ public byte[] encodeLong(long l) {
+ return encodeLong(l, new byte[16]);
+ }
+
+ @Override
+ public byte[] encodeLong(long l, byte[] ret) {
+ encode(ret, l);
+ return ret;
+ }
+
+ @Override
+ public long decodeLong(byte[] a) {
+ return decode(a);
+ }
+
+ @Override
+ public byte[] encodeDouble(double d) {
+ return encodeDouble(d, new byte[16]);
+ }
+
+ @Override
+ public byte[] encodeDouble(double d, byte[] ret) {
+ return encodeLong(Double.doubleToRawLongBits(d), ret);
+ }
+
+ @Override
+ public double decodeDouble(byte[] a) {
+ return Double.longBitsToDouble(decodeLong(a));
+ }
+
+ @Override
+ public byte[] encodeFloat(float d) {
+ return encodeFloat(d, new byte[16]);
+ }
+
+ @Override
+ public byte[] encodeFloat(float d, byte[] ret) {
+ return encodeInt(Float.floatToRawIntBits(d), ret);
+ }
+
+ @Override
+ public float decodeFloat(byte[] a) {
+ return Float.intBitsToFloat(decodeInt(a));
+ }
+
+ @Override
+ public boolean decodeBoolean(byte[] val) {
+ return decodeByte(val) == 1;
+ }
+
+ @Override
+ public byte[] encodeBoolean(boolean b) {
+ return encodeBoolean(b, new byte[2]);
+ }
+
+ @Override
+ public byte[] encodeBoolean(boolean b, byte[] ret) {
+ if (b)
+ encode(ret, 1);
+ else
+ encode(ret, 0);
+
+ return ret;
+ }
+
+ private byte[] toBinary(byte[] hex) {
+ byte[] bin = new byte[(hex.length / 2) + (hex.length % 2)];
+
+ int j = 0;
+ for (int i = 0; i < bin.length; i++) {
+ bin[i] = (byte) (fromChar(hex[j++]) << 4);
+ if (j >= hex.length)
+ break;
+ bin[i] |= (byte) fromChar(hex[j++]);
+ }
+
+ return bin;
+ }
+
+ private byte[] fromBinary(byte[] bin) {
+ byte[] hex = new byte[bin.length * 2];
+
+ int j = 0;
+ for (byte aBin : bin) {
+ hex[j++] = chars[0x0f & (aBin >>> 4)];
+ hex[j++] = chars[0x0f & aBin];
+ }
+
+ return hex;
+ }
+
+ @Override
+ public byte[] followingKey(int size, byte[] per) {
+ return fromBinary(Utils.followingKey(size, toBinary(per)));
+ }
+
+ @Override
+ public byte[] lastPossibleKey(int size, byte[] er) {
+ return fromBinary(Utils.lastPossibleKey(size, toBinary(er)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java
new file mode 100644
index 0000000..a8216f4
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.encoders;
+
+import java.io.IOException;
+
+/**
+ * This class transforms this bits within a primitive type so that
+ * the bit representation sorts correctly lexographicaly. Primarily
+ * it does some simple transformations so that negative numbers sort
+ * before positive numbers, when compared lexographically.
+ */
+public class SignedBinaryEncoder extends BinaryEncoder {
+
+ @Override
+ public byte[] encodeShort(short s, byte[] ret) throws IOException{
+ s = (short)((s & 0xffff) ^ 0x8000);
+ return super.encodeShort(s, ret);
+ }
+
+ @Override
+ public short decodeShort(byte[] a) throws IOException{
+ short s = super.decodeShort(a);
+ s = (short)((s & 0xffff) ^ 0x8000);
+ return s;
+ }
+
+ @Override
+ public byte[] encodeInt(int i, byte[] ret) throws IOException{
+ i = i ^ 0x80000000;
+ return super.encodeInt(i, ret);
+ }
+
+ @Override
+ public int decodeInt(byte[] a) throws IOException{
+ int i = super.decodeInt(a);
+ i = i ^ 0x80000000;
+ return i;
+ }
+
+ @Override
+ public byte[] encodeLong(long l, byte[] ret) throws IOException{
+ l = l ^ 0x8000000000000000L;
+ return super.encodeLong(l, ret);
+ }
+
+ @Override
+ public long decodeLong(byte[] a) throws IOException {
+ long l = super.decodeLong(a);
+ l = l ^ 0x8000000000000000L;
+ return l;
+ }
+
+ @Override
+ public byte[] encodeDouble(double d, byte[] ret) throws IOException {
+ long l = Double.doubleToRawLongBits(d);
+ if(l < 0)
+ l = ~l;
+ else
+ l = l ^ 0x8000000000000000L;
+ return super.encodeLong(l,ret);
+ }
+
+ @Override
+ public double decodeDouble(byte[] a) throws IOException{
+ long l = super.decodeLong(a);
+ if(l < 0)
+ l = l ^ 0x8000000000000000L;
+ else
+ l = ~l;
+ return Double.longBitsToDouble(l);
+ }
+
+ @Override
+ public byte[] encodeFloat(float f, byte[] ret) throws IOException {
+ int i = Float.floatToRawIntBits(f);
+ if(i < 0)
+ i = ~i;
+ else
+ i = i ^ 0x80000000;
+
+ return super.encodeInt(i, ret);
+
+ }
+
+ @Override
+ public float decodeFloat(byte[] a) throws IOException{
+ int i = super.decodeInt(a);
+ if(i < 0)
+ i = i ^ 0x80000000;
+ else
+ i = ~i;
+ return Float.intBitsToFloat(i);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java
new file mode 100644
index 0000000..8a5980c
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.encoders;
+
+import java.math.BigInteger;
+import java.util.Arrays;
+
+/**
+ *
+ */
+public class Utils {
+ private static BigInteger newPositiveBigInteger(byte[] er) {
+ byte[] copy = new byte[er.length + 1];
+ System.arraycopy(er, 0, copy, 1, er.length);
+ BigInteger bi = new BigInteger(copy);
+ return bi;
+ }
+
+ public static byte[] lastPossibleKey(int size, byte[] er) {
+ if (size == er.length)
+ return er;
+
+ if (er.length > size)
+ throw new IllegalArgumentException();
+
+ BigInteger bi = newPositiveBigInteger(er);
+ if (bi.equals(BigInteger.ZERO))
+ throw new IllegalArgumentException("Nothing comes before zero");
+
+ bi = bi.subtract(BigInteger.ONE);
+
+ byte[] ret = new byte[size];
+ Arrays.fill(ret, (byte) 0xff);
+
+ System.arraycopy(getBytes(bi, er.length), 0, ret, 0, er.length);
+
+ return ret;
+ }
+
+ private static byte[] getBytes(BigInteger bi, int minLen) {
+ byte[] ret = bi.toByteArray();
+
+ if (ret[0] == 0) {
+ // remove leading 0 that makes num positive
+ byte[] copy = new byte[ret.length - 1];
+ System.arraycopy(ret, 1, copy, 0, copy.length);
+ ret = copy;
+ }
+
+ // leading digits are dropped
+ byte[] copy = new byte[minLen];
+ if (bi.compareTo(BigInteger.ZERO) < 0) {
+ Arrays.fill(copy, (byte) 0xff);
+ }
+ System.arraycopy(ret, 0, copy, minLen - ret.length, ret.length);
+
+ return copy;
+ }
+
+ public static byte[] followingKey(int size, byte[] per) {
+
+ if (per.length > size)
+ throw new IllegalArgumentException();
+
+ if (size == per.length) {
+ // add one
+ BigInteger bi = new BigInteger(per);
+ bi = bi.add(BigInteger.ONE);
+ if (bi.equals(BigInteger.ZERO)) {
+ throw new IllegalArgumentException("Wrapped");
+ }
+ return getBytes(bi, size);
+ } else {
+ return Arrays.copyOf(per, size);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java
new file mode 100644
index 0000000..574aa24
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite store related util classes for encoder.
+ */
+package org.apache.gora.ignite.encoders;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
new file mode 100644
index 0000000..a7fa7ab
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite datastore related all classes.
+ */
+package org.apache.gora.ignite;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
new file mode 100644
index 0000000..85a59c9
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.query;
+
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.store.DataStore;
+
+/**
+ * Ignite specific implementation of the {@link org.apache.gora.query.Query} interface.
+ */
+public class IgniteQuery<K,T extends PersistentBase> extends QueryBase<K,T> {
+
+ /**
+ * Constructor for the query
+ */
+ public IgniteQuery() {
+ super(null);
+ }
+
+ /**
+ * Constructor for the query
+ *
+ * @param dataStore Data store used
+ *
+ */
+ public IgniteQuery(DataStore<K,T> dataStore) {
+ super(dataStore);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
new file mode 100644
index 0000000..416e650
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.query;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.ignite.core.client.RowIterator;
+import org.apache.ignite.core.client.Scanner;
+import org.apache.ignite.core.data.ByteSequence;
+import org.apache.ignite.core.data.Key;
+import org.apache.ignite.core.data.Value;
+import org.apache.gora.ignite.store.IgniteStore;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.impl.ResultBase;
+import org.apache.gora.store.DataStore;
+
+/**
+ * Ignite specific implementation of the {@link org.apache.gora.query.Result} interface.
+ */
+public class IgniteResult<K,T extends PersistentBase> extends ResultBase<K,T> {
+
+ private RowIterator iterator;
+
+ /**
+ * Gets the data store used
+ */
+ public IgniteStore<K,T> getDataStore() {
+ return (IgniteStore<K,T>) super.getDataStore();
+ }
+
+ /**
+ * @param dataStore
+ * @param query
+ * @param scanner
+ */
+ public IgniteResult(DataStore<K,T> dataStore, Query<K,T> query, Scanner scanner) {
+ super(dataStore, query);
+
+ if (this.limit > 0) {
+ scanner.setBatchSize((int) this.limit);
+ }
+ iterator = new RowIterator(scanner.iterator());
+ }
+
+ /**
+ * Gets the items reading progress
+ */
+ @Override
+ public float getProgress() throws IOException {
+ if (this.limit != -1) {
+ return (float) this.offset / (float) this.limit;
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ /**
+ * Gets the next item
+ */
+ @Override
+ protected boolean nextInner() throws IOException {
+
+ if (!iterator.hasNext())
+ return false;
+
+ key = null;
+
+ Iterator<Entry<Key,Value>> nextRow = iterator.next();
+ ByteSequence row = getDataStore().populate(nextRow, persistent);
+ key = ((IgniteStore<K, T>) dataStore).fromBytes(getKeyClass(), row.toArray());
+
+ return true;
+ }
+
+ @Override
+ public int size() {
+ return (int) this.limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
new file mode 100644
index 0000000..b1a306c
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains all the Ignite store query representation class as well as Result set representing class
+ * when query is executed over the Ignite dataStore.
+ */
+package org.apache.gora.ignite.query;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
new file mode 100644
index 0000000..b46c063
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ignite.core.util.Pair;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Mapping definitions for Ignite.
+ */
+public class IgniteMapping {
+
+ /**
+ * A map of field names to Field objects containing schema's fields
+ */
+ Map<String,Pair<Text,Text>> fieldMap = new HashMap<>();
+
+ /**
+ * Look up the column associated to the Avro field.
+ */
+ Map<Pair<Text,Text>,String> columnMap = new HashMap<>();
+
+ Map<String,String> tableConfig = new HashMap<>();
+ String tableName;
+ String encoder;
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
new file mode 100644
index 0000000..2f5faf9
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -0,0 +1,1034 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+
+import org.apache.ignite.core.client.IgniteException;
+import org.apache.ignite.core.client.IgniteSecurityException;
+import org.apache.ignite.core.client.BatchWriter;
+import org.apache.ignite.core.client.BatchWriterConfig;
+import org.apache.ignite.core.client.Connector;
+import org.apache.ignite.core.client.IsolatedScanner;
+import org.apache.ignite.core.client.IteratorSetting;
+import org.apache.ignite.core.client.MutationsRejectedException;
+import org.apache.ignite.core.client.RowIterator;
+import org.apache.ignite.core.client.Scanner;
+import org.apache.ignite.core.client.TableDeletedException;
+import org.apache.ignite.core.client.TableExistsException;
+import org.apache.ignite.core.client.TableNotFoundException;
+import org.apache.ignite.core.client.TableOfflineException;
+import org.apache.ignite.core.client.ZooKeeperInstance;
+import org.apache.ignite.core.client.impl.ClientContext;
+import org.apache.ignite.core.client.impl.Tables;
+import org.apache.ignite.core.client.impl.TabletLocator;
+import org.apache.ignite.core.client.mock.MockConnector;
+import org.apache.ignite.core.client.mock.MockInstance;
+import org.apache.ignite.core.client.mock.impl.MockTabletLocator;
+import org.apache.ignite.core.client.security.tokens.AuthenticationToken;
+import org.apache.ignite.core.client.security.tokens.PasswordToken;
+import org.apache.ignite.core.conf.IgniteConfiguration;
+import org.apache.ignite.core.data.ByteSequence;
+import org.apache.ignite.core.data.Key;
+import org.apache.ignite.core.data.impl.KeyExtent;
+import org.apache.ignite.core.data.Mutation;
+import org.apache.ignite.core.data.Range;
+import org.apache.ignite.core.data.Value;
+import org.apache.ignite.core.iterators.SortedKeyIterator;
+import org.apache.ignite.core.iterators.user.TimestampFilter;
+import org.apache.ignite.core.master.state.tables.TableState;
+import org.apache.ignite.core.security.Authorizations;
+import org.apache.ignite.core.security.ColumnVisibility;
+import org.apache.ignite.core.client.impl.Credentials;
+import org.apache.ignite.core.util.Pair;
+import org.apache.ignite.core.util.UtilWaitThread;
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.io.BinaryDecoder;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.avro.util.Utf8;
+import org.apache.gora.ignite.encoders.BinaryEncoder;
+import org.apache.gora.ignite.encoders.Encoder;
+import org.apache.gora.ignite.query.IgniteQuery;
+import org.apache.gora.ignite.query.IgniteResult;
+import org.apache.gora.persistency.impl.DirtyListWrapper;
+import org.apache.gora.persistency.impl.DirtyMapWrapper;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.PartitionQuery;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.Result;
+import org.apache.gora.query.impl.PartitionQueryImpl;
+import org.apache.gora.store.DataStoreFactory;
+import org.apache.gora.store.impl.DataStoreBase;
+import org.apache.gora.util.AvroUtils;
+import org.apache.gora.util.GoraException;
+import org.apache.gora.util.IOUtils;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+/**
+ * Implementation of a Ignite data store to be used by gora.
+ *
+ * @param <K>
+ * class to be used for the key
+ * @param <T>
+ * class to be persisted within the store
+ */
+public class IgniteStore<K,T extends PersistentBase> extends DataStoreBase<K,T> {
+
+ protected static final String MOCK_PROPERTY = "ignite.mock";
+ protected static final String INSTANCE_NAME_PROPERTY = "ignite.instance";
+ protected static final String ZOOKEEPERS_NAME_PROPERTY = "ignite.zookeepers";
+ protected static final String USERNAME_PROPERTY = "ignite.user";
+ protected static final String PASSWORD_PROPERTY = "ignite.password";
+ protected static final String DEFAULT_MAPPING_FILE = "gora-ignite-mapping.xml";
+
+ private final static String UNKOWN = "Unknown type ";
+
+ private Connector conn;
+ private BatchWriter batchWriter;
+ private IgniteMapping mapping;
+ private Credentials credentials;
+ private Encoder encoder;
+
+ public static final Logger LOG = LoggerFactory.getLogger(IgniteStore.class);
+
+ public Object fromBytes(Schema schema, byte[] data) throws IOException {
+ Schema fromSchema = null;
+ if (schema.getType() == Type.UNION) {
+ try {
+ Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
+ int unionIndex = decoder.readIndex();
+ List<Schema> possibleTypes = schema.getTypes();
+ fromSchema = possibleTypes.get(unionIndex);
+ Schema effectiveSchema = possibleTypes.get(unionIndex);
+ if (effectiveSchema.getType() == Type.NULL) {
+ decoder.readNull();
+ return null;
+ } else {
+ data = decoder.readBytes(null).array();
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage());
+ throw new GoraException("Error decoding union type: ", e);
+ }
+ } else {
+ fromSchema = schema;
+ }
+ return fromBytes(encoder, fromSchema, data);
+ }
+
+ public static Object fromBytes(Encoder encoder, Schema schema, byte data[]) throws IOException {
+ switch (schema.getType()) {
+ case BOOLEAN:
+ return encoder.decodeBoolean(data);
+ case DOUBLE:
+ return encoder.decodeDouble(data);
+ case FLOAT:
+ return encoder.decodeFloat(data);
+ case INT:
+ return encoder.decodeInt(data);
+ case LONG:
+ return encoder.decodeLong(data);
+ case STRING:
+ return new Utf8(data);
+ case BYTES:
+ return ByteBuffer.wrap(data);
+ case ENUM:
+ return AvroUtils.getEnumValue(schema, encoder.decodeInt(data));
+ case ARRAY:
+ break;
+ case FIXED:
+ break;
+ case MAP:
+ break;
+ case NULL:
+ break;
+ case RECORD:
+ break;
+ case UNION:
+ break;
+ default:
+ break;
+ }
+ throw new IllegalArgumentException(UNKOWN + schema.getType());
+
+ }
+
+ private static byte[] getBytes(Text text) {
+ byte[] bytes = text.getBytes();
+ if (bytes.length != text.getLength()) {
+ bytes = new byte[text.getLength()];
+ System.arraycopy(text.getBytes(), 0, bytes, 0, bytes.length);
+ }
+ return bytes;
+ }
+
+ public K fromBytes(Class<K> clazz, byte[] val) {
+ return fromBytes(encoder, clazz, val);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <K> K fromBytes(Encoder encoder, Class<K> clazz, byte[] val) {
+ try {
+ if (clazz.equals(Byte.TYPE) || clazz.equals(Byte.class)) {
+ return (K) Byte.valueOf(encoder.decodeByte(val));
+ } else if (clazz.equals(Boolean.TYPE) || clazz.equals(Boolean.class)) {
+ return (K) Boolean.valueOf(encoder.decodeBoolean(val));
+ } else if (clazz.equals(Short.TYPE) || clazz.equals(Short.class)) {
+ return (K) Short.valueOf(encoder.decodeShort(val));
+ } else if (clazz.equals(Integer.TYPE) || clazz.equals(Integer.class)) {
+ return (K) Integer.valueOf(encoder.decodeInt(val));
+ } else if (clazz.equals(Long.TYPE) || clazz.equals(Long.class)) {
+ return (K) Long.valueOf(encoder.decodeLong(val));
+ } else if (clazz.equals(Float.TYPE) || clazz.equals(Float.class)) {
+ return (K) Float.valueOf(encoder.decodeFloat(val));
+ } else if (clazz.equals(Double.TYPE) || clazz.equals(Double.class)) {
+ return (K) Double.valueOf(encoder.decodeDouble(val));
+ } else if (clazz.equals(String.class)) {
+ return (K) new String(val, "UTF-8");
+ } else if (clazz.equals(Utf8.class)) {
+ return (K) new Utf8(val);
+ }
+
+ throw new IllegalArgumentException(UNKOWN + clazz.getName());
+ } catch (IOException ioe) {
+ LOG.error(ioe.getMessage());
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ private static byte[] copyIfNeeded(byte b[], int offset, int len) {
+ if (len != b.length || offset != 0) {
+ byte[] copy = new byte[len];
+ System.arraycopy(b, offset, copy, 0, copy.length);
+ b = copy;
+ }
+ return b;
+ }
+
+ public byte[] toBytes(Schema toSchema, Object o) {
+ if (toSchema != null && toSchema.getType() == Type.UNION) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ org.apache.avro.io.BinaryEncoder avroEncoder = EncoderFactory.get().binaryEncoder(baos, null);
+ int unionIndex = 0;
+ try {
+ if (o == null) {
+ unionIndex = firstNullSchemaTypeIndex(toSchema);
+ avroEncoder.writeIndex(unionIndex);
+ avroEncoder.writeNull();
+ } else {
+ unionIndex = firstNotNullSchemaTypeIndex(toSchema);
+ avroEncoder.writeIndex(unionIndex);
+ avroEncoder.writeBytes(toBytes(o));
+ }
+ avroEncoder.flush();
+ return baos.toByteArray();
+ } catch (IOException e) {
+ LOG.error(e.getMessage());
+ return toBytes(o);
+ }
+ } else {
+ return toBytes(o);
+ }
+ }
+
+ private int firstNullSchemaTypeIndex(Schema toSchema) {
+ List<Schema> possibleTypes = toSchema.getTypes();
+ int unionIndex = 0;
+ for (int i = 0; i < possibleTypes.size(); i++ ) {
+ Type pType = possibleTypes.get(i).getType();
+ if (pType == Type.NULL) { // FIXME HUGE kludge to pass tests
+ unionIndex = i; break;
+ }
+ }
+ return unionIndex;
+ }
+
+ private int firstNotNullSchemaTypeIndex(Schema toSchema) {
+ List<Schema> possibleTypes = toSchema.getTypes();
+ int unionIndex = 0;
+ for (int i = 0; i < possibleTypes.size(); i++ ) {
+ Type pType = possibleTypes.get(i).getType();
+ if (pType != Type.NULL) { // FIXME HUGE kludge to pass tests
+ unionIndex = i; break;
+ }
+ }
+ return unionIndex;
+ }
+
+ public byte[] toBytes(Object o) {
+ return toBytes(encoder, o);
+ }
+
+ public static byte[] toBytes(Encoder encoder, Object o) {
+
+ try {
+ if (o instanceof String) {
+ return ((String) o).getBytes("UTF-8");
+ } else if (o instanceof Utf8) {
+ return copyIfNeeded(((Utf8) o).getBytes(), 0, ((Utf8) o).getByteLength());
+ } else if (o instanceof ByteBuffer) {
+ return copyIfNeeded(((ByteBuffer) o).array(), ((ByteBuffer) o).arrayOffset() + ((ByteBuffer) o).position(), ((ByteBuffer) o).remaining());
+ } else if (o instanceof Long) {
+ return encoder.encodeLong((Long) o);
+ } else if (o instanceof Integer) {
+ return encoder.encodeInt((Integer) o);
+ } else if (o instanceof Short) {
+ return encoder.encodeShort((Short) o);
+ } else if (o instanceof Byte) {
+ return encoder.encodeByte((Byte) o);
+ } else if (o instanceof Boolean) {
+ return encoder.encodeBoolean((Boolean) o);
+ } else if (o instanceof Float) {
+ return encoder.encodeFloat((Float) o);
+ } else if (o instanceof Double) {
+ return encoder.encodeDouble((Double) o);
+ } else if (o instanceof Enum) {
+ return encoder.encodeInt(((Enum<?>) o).ordinal());
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ throw new IllegalArgumentException(UNKOWN + o.getClass().getName());
+ }
+
+ private BatchWriter getBatchWriter() throws IOException {
+ if (batchWriter == null)
+ try {
+ BatchWriterConfig batchWriterConfig = new BatchWriterConfig();
+ batchWriterConfig.setMaxMemory(10000000);
+ batchWriterConfig.setMaxLatency(60000L, TimeUnit.MILLISECONDS);
+ batchWriterConfig.setMaxWriteThreads(4);
+ batchWriter = conn.createBatchWriter(mapping.tableName, batchWriterConfig);
+ } catch (TableNotFoundException e) {
+ throw new IOException(e);
+ }
+ return batchWriter;
+ }
+
+ /**
+ * Initialize the data store by reading the credentials, setting the client's properties up and
+ * reading the mapping file. Initialize is called when then the call to
+ * {@link org.apache.gora.store.DataStoreFactory#createDataStore} is made.
+ *
+ * @param keyClass
+ * @param persistentClass
+ * @param properties
+ */
+ @Override
+ public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) throws GoraException {
+ super.initialize(keyClass, persistentClass, properties);
+
+ try {
+
+ String mock = DataStoreFactory.findProperty(properties, this, MOCK_PROPERTY, null);
+ String mappingFile = DataStoreFactory.getMappingFile(properties, this, DEFAULT_MAPPING_FILE);
+ String user = DataStoreFactory.findProperty(properties, this, USERNAME_PROPERTY, null);
+ String password = DataStoreFactory.findProperty(properties, this, PASSWORD_PROPERTY, null);
+
+ mapping = readMapping(mappingFile);
+
+ if (mapping.encoder == null || "".equals(mapping.encoder)) {
+ encoder = new BinaryEncoder();
+ } else {
+ encoder = (Encoder) getClass().getClassLoader().loadClass(mapping.encoder).newInstance();
+ }
+
+ AuthenticationToken token = new PasswordToken(password);
+ if (mock == null || !mock.equals("true")) {
+ String instance = DataStoreFactory.findProperty(properties, this, INSTANCE_NAME_PROPERTY, null);
+ String zookeepers = DataStoreFactory.findProperty(properties, this, ZOOKEEPERS_NAME_PROPERTY, null);
+ conn = new ZooKeeperInstance(instance, zookeepers).getConnector(user, token);
+ } else {
+ conn = new MockInstance().getConnector(user, token);
+ }
+ credentials = new Credentials(user, token);
+
+ if (autoCreateSchema && !schemaExists())
+ createSchema();
+
+ } catch (IOException | InstantiationException | IllegalAccessException |
+ ClassNotFoundException | IgniteException | IgniteSecurityException e) {
+ throw new GoraException(e);
+ }
+ }
+
+ protected IgniteMapping readMapping(String filename) throws IOException {
+ try {
+
+ IgniteMapping mapping = new IgniteMapping();
+
+ DocumentBuilder db = DocumentBuilderFactory.newInstance().newDocumentBuilder();
+ Document dom = db.parse(getClass().getClassLoader().getResourceAsStream(filename));
+
+ Element root = dom.getDocumentElement();
+
+ NodeList nl = root.getElementsByTagName("class");
+ for (int i = 0; i < nl.getLength(); i++) {
+
+ Element classElement = (Element) nl.item(i);
+ if (classElement.getAttribute("keyClass").equals(keyClass.getCanonicalName())
+ && classElement.getAttribute("name").equals(persistentClass.getCanonicalName())) {
+
+ mapping.tableName = getSchemaName(classElement.getAttribute("table"), persistentClass);
+ mapping.encoder = classElement.getAttribute("encoder");
+
+ NodeList fields = classElement.getElementsByTagName("field");
+ for (int j = 0; j < fields.getLength(); j++) {
+ Element fieldElement = (Element) fields.item(j);
+
+ String name = fieldElement.getAttribute("name");
+ String family = fieldElement.getAttribute("family");
+ String qualifier = fieldElement.getAttribute("qualifier");
+ if ("".equals(qualifier))
+ qualifier = null;
+
+ Pair<Text,Text> col = new Pair<>(new Text(family), qualifier == null ? null : new Text(qualifier));
+ mapping.fieldMap.put(name, col);
+ mapping.columnMap.put(col, name);
+ }
+ }
+
+ }
+
+ if (mapping.tableName == null) {
+ throw new GoraException("Please define the ignite 'table' name mapping in " + filename + " for " + persistentClass.getCanonicalName());
+ }
+
+ nl = root.getElementsByTagName("table");
+ for (int i = 0; i < nl.getLength(); i++) {
+ Element tableElement = (Element) nl.item(i);
+ if (tableElement.getAttribute("name").equals(mapping.tableName)) {
+ NodeList configs = tableElement.getElementsByTagName("config");
+ for (int j = 0; j < configs.getLength(); j++) {
+ Element configElement = (Element) configs.item(j);
+ String key = configElement.getAttribute("key");
+ String val = configElement.getAttribute("value");
+ mapping.tableConfig.put(key, val);
+ }
+ }
+ }
+
+ return mapping;
+ } catch (Exception ex) {
+ throw new IOException("Unable to read " + filename, ex);
+ }
+
+ }
+
+ @Override
+ public String getSchemaName() {
+ return mapping.tableName;
+ }
+
+ @Override
+ public void createSchema() throws GoraException {
+ try {
+ conn.tableOperations().create(mapping.tableName);
+ Set<Entry<String,String>> es = mapping.tableConfig.entrySet();
+ for (Entry<String,String> entry : es) {
+ conn.tableOperations().setProperty(mapping.tableName, entry.getKey(), entry.getValue());
+ }
+
+ } catch (TableExistsException e) {
+ LOG.debug(e.getMessage(), e);
+ // Assume this is not an error
+ } catch (IgniteException | IgniteSecurityException e) {
+ throw new GoraException(e);
+ }
+ }
+
+ @Override
+ public void deleteSchema() throws GoraException {
+ try {
+ if (batchWriter != null)
+ batchWriter.close();
+ batchWriter = null;
+ conn.tableOperations().delete(mapping.tableName);
+ } catch (TableNotFoundException e) {
+ // Ignore. Delete a non existant schema is a success
+ } catch (IgniteException | IgniteSecurityException e) {
+ throw new GoraException(e);
+ }
+ }
+
+ @Override
+ public boolean schemaExists() throws GoraException {
+ try {
+ return conn.tableOperations().exists(mapping.tableName);
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
+ }
+
+ public ByteSequence populate(Iterator<Entry<Key,Value>> iter, T persistent) throws IOException {
+ ByteSequence row = null;
+
+ Map<Utf8, Object> currentMap = null;
+ List currentArray = null;
+ Text currentFam = null;
+ int currentPos = 0;
+ Schema currentSchema = null;
+ Field currentField = null;
+
+ BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(new byte[0], null);
+
+ while (iter.hasNext()) {
+ Entry<Key,Value> entry = iter.next();
+
+ if (row == null) {
+ row = entry.getKey().getRowData();
+ }
+ byte[] val = entry.getValue().get();
+
+ Field field = fieldMap.get(getFieldName(entry));
+
+ if (currentMap != null) {
+ if (currentFam.equals(entry.getKey().getColumnFamily())) {
+ currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
+ fromBytes(currentSchema, entry.getValue().get()));
+ continue;
+ } else {
+ persistent.put(currentPos, currentMap);
+ currentMap = null;
+ }
+ } else if (currentArray != null) {
+ if (currentFam.equals(entry.getKey().getColumnFamily())) {
+ currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
+ continue;
+ } else {
+ persistent.put(currentPos, new GenericData.Array<T>(currentField.schema(), currentArray));
+ currentArray = null;
+ }
+ }
+
+ switch (field.schema().getType()) {
+ case MAP: // first entry only. Next are handled above on the next loop
+ currentMap = new DirtyMapWrapper<>(new HashMap<Utf8, Object>());
+ currentPos = field.pos();
+ currentFam = entry.getKey().getColumnFamily();
+ currentSchema = field.schema().getValueType();
+
+ currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
+ fromBytes(currentSchema, entry.getValue().get()));
+ break;
+ case ARRAY:
+ currentArray = new DirtyListWrapper<>(new ArrayList<>());
+ currentPos = field.pos();
+ currentFam = entry.getKey().getColumnFamily();
+ currentSchema = field.schema().getElementType();
+ currentField = field;
+
+ currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
+
+ break;
+ case UNION:// default value of null acts like union with null
+ Schema effectiveSchema = field.schema().getTypes()
+ .get(firstNotNullSchemaTypeIndex(field.schema()));
+ // map and array were coded without union index so need to be read the same way
+ if (effectiveSchema.getType() == Type.ARRAY) {
+ currentArray = new DirtyListWrapper<>(new ArrayList<>());
+ currentPos = field.pos();
+ currentFam = entry.getKey().getColumnFamily();
+ currentSchema = field.schema().getElementType();
+ currentField = field;
+
+ currentArray.add(fromBytes(currentSchema, entry.getValue().get()));
+ break;
+ }
+ else if (effectiveSchema.getType() == Type.MAP) {
+ currentMap = new DirtyMapWrapper<>(new HashMap<Utf8, Object>());
+ currentPos = field.pos();
+ currentFam = entry.getKey().getColumnFamily();
+ currentSchema = effectiveSchema.getValueType();
+
+ currentMap.put(new Utf8(entry.getKey().getColumnQualifierData().toArray()),
+ fromBytes(currentSchema, entry.getValue().get()));
+ break;
+ }
+ // continue like a regular top-level union
+ case RECORD:
+ SpecificDatumReader<?> reader = new SpecificDatumReader<Schema>(field.schema());
+ persistent.put(field.pos(), reader.read(null, DecoderFactory.get().binaryDecoder(val, decoder)));
+ break;
+ default:
+ persistent.put(field.pos(), fromBytes(field.schema(), entry.getValue().get()));
+ }
+ }
+
+ if (currentMap != null) {
+ persistent.put(currentPos, currentMap);
+ } else if (currentArray != null) {
+ persistent.put(currentPos, new GenericData.Array<T>(currentField.schema(), currentArray));
+ }
+
+ persistent.clearDirty();
+
+ return row;
+ }
+
+ /**
+ * Retrieve field name from entry.
+ * @param entry The Key-Value entry
+ * @return String The field name
+ */
+ private String getFieldName(Entry<Key, Value> entry) {
+ String fieldName = mapping.columnMap.get(new Pair<>(entry.getKey().getColumnFamily(),
+ entry.getKey().getColumnQualifier()));
+ if (fieldName == null) {
+ fieldName = mapping.columnMap.get(new Pair<Text,Text>(entry.getKey().getColumnFamily(), null));
+ }
+ return fieldName;
+ }
+
+ private void setFetchColumns(Scanner scanner, String[] fields) {
+ fields = getFieldsToQuery(fields);
+ for (String field : fields) {
+ Pair<Text,Text> col = mapping.fieldMap.get(field);
+ if (col != null) {
+ if (col.getSecond() == null) {
+ scanner.fetchColumnFamily(col.getFirst());
+ } else {
+ scanner.fetchColumn(col.getFirst(), col.getSecond());
+ }
+ } else {
+ LOG.error("Mapping not found for field: {}", field);
+ }
+ }
+ }
+
+ @Override
+ public T get(K key, String[] fields) throws GoraException {
+ try {
+ // TODO make isolated scanner optional?
+ Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
+ Range rowRange = new Range(new Text(toBytes(key)));
+
+ scanner.setRange(rowRange);
+ setFetchColumns(scanner, fields);
+
+ T persistent = newPersistent();
+ ByteSequence row = populate(scanner.iterator(), persistent);
+ if (row == null)
+ return null;
+ return persistent;
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
+ }
+
+ @Override
+ public void put(K key, T val) throws GoraException {
+
+ try{
+ Mutation m = new Mutation(new Text(toBytes(key)));
+
+ Schema schema = val.getSchema();
+ List<Field> fields = schema.getFields();
+ int count = 0;
+
+ for (int i = 0; i < fields.size(); i++) {
+ if (!val.isDirty(i)) {
+ continue;
+ }
+ Field field = fields.get(i);
+
+ Object o = val.get(field.pos());
+
+ Pair<Text,Text> col = mapping.fieldMap.get(field.name());
+
+ if (col == null) {
+ throw new GoraException("Please define the gora to ignite mapping for field " + field.name());
+ }
+
+ switch (field.schema().getType()) {
+ case MAP:
+ count = putMap(m, count, field.schema().getValueType(), o, col, field.name());
+ break;
+ case ARRAY:
+ count = putArray(m, count, o, col, field.name());
+ break;
+ case UNION: // default value of null acts like union with null
+ Schema effectiveSchema = field.schema().getTypes()
+ .get(firstNotNullSchemaTypeIndex(field.schema()));
+ // map and array need to compute qualifier
+ if (effectiveSchema.getType() == Type.ARRAY) {
+ count = putArray(m, count, o, col, field.name());
+ break;
+ }
+ else if (effectiveSchema.getType() == Type.MAP) {
+ count = putMap(m, count, effectiveSchema.getValueType(), o, col, field.name());
+ break;
+ }
+ // continue like a regular top-level union
+ case RECORD:
+ final SpecificDatumWriter<Object> writer = new SpecificDatumWriter<>(field.schema());
+ final byte[] byteData = IOUtils.serialize(writer,o);
+ m.put(col.getFirst(), col.getSecond(), new Value(byteData));
+ count++;
+ break;
+ default:
+ m.put(col.getFirst(), col.getSecond(), new Value(toBytes(o)));
+ count++;
+ }
+
+ }
+
+ if (count > 0)
+ try {
+ getBatchWriter().addMutation(m);
+ } catch (MutationsRejectedException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ } catch (GoraException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
+ }
+
+ private int putMap(Mutation m, int count, Schema valueType, Object o, Pair<Text, Text> col, String fieldName) throws GoraException {
+
+ // First of all we delete map field on ignite store
+ Text rowKey = new Text(m.getRow());
+ Query<K, T> query = newQuery();
+ query.setFields(fieldName);
+ query.setStartKey((K)rowKey.toString());
+ query.setEndKey((K)rowKey.toString());
+ deleteByQuery(query);
+ flush();
+ if (o == null){
+ return 0;
+ }
+
+ Set<?> es = ((Map<?, ?>)o).entrySet();
+ for (Object entry : es) {
+ Object mapKey = ((Entry<?, ?>) entry).getKey();
+ Object mapVal = ((Entry<?, ?>) entry).getValue();
+ if ((o instanceof DirtyMapWrapper && ((DirtyMapWrapper<?, ?>)o).isDirty())
+ || !(o instanceof DirtyMapWrapper)) {
+ m.put(col.getFirst(), new Text(toBytes(mapKey)), new Value(toBytes(valueType, mapVal)));
+ count++;
+ }
+ // TODO map value deletion
+ }
+ return count;
+ }
+
+ private int putArray(Mutation m, int count, Object o, Pair<Text, Text> col, String fieldName) throws GoraException {
+
+ // First of all we delete array field on ignite store
+ Text rowKey = new Text(m.getRow());
+ Query<K, T> query = newQuery();
+ query.setFields(fieldName);
+ query.setStartKey((K)rowKey.toString());
+ query.setEndKey((K)rowKey.toString());
+ deleteByQuery(query);
+ flush();
+ if (o == null){
+ return 0;
+ }
+
+ List<?> array = (List<?>) o; // both GenericArray and DirtyListWrapper
+ int j = 0;
+ for (Object item : array) {
+ m.put(col.getFirst(), new Text(toBytes(j++)), new Value(toBytes(item)));
+ count++;
+ }
+ return count;
+ }
+
+ @Override
+ public boolean delete(K key) throws GoraException {
+ Query<K,T> q = newQuery();
+ q.setKey(key);
+ return deleteByQuery(q) > 0;
+ }
+
+ @Override
+ public long deleteByQuery(Query<K,T> query) throws GoraException {
+ try {
+ Scanner scanner = createScanner(query);
+ // add iterator that drops values on the server side
+ scanner.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class));
+ RowIterator iterator = new RowIterator(scanner.iterator());
+
+ long count = 0;
+
+ while (iterator.hasNext()) {
+ Iterator<Entry<Key,Value>> row = iterator.next();
+ Mutation m = null;
+ while (row.hasNext()) {
+ Entry<Key,Value> entry = row.next();
+ Key key = entry.getKey();
+ if (m == null)
+ m = new Mutation(key.getRow());
+ // TODO optimize to avoid continually creating column vis? prob does not matter for empty
+ m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp());
+ }
+ getBatchWriter().addMutation(m);
+ count++;
+ }
+
+ return count;
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
+ }
+
+ private Range createRange(Query<K,T> query) {
+ Text startRow = null;
+ Text endRow = null;
+
+ if (query.getStartKey() != null)
+ startRow = new Text(toBytes(query.getStartKey()));
+
+ if (query.getEndKey() != null)
+ endRow = new Text(toBytes(query.getEndKey()));
+
+ return new Range(startRow, true, endRow, true);
+
+ }
+
+ private Scanner createScanner(Query<K,T> query) throws TableNotFoundException {
+ // TODO make isolated scanner optional?
+ Scanner scanner = new IsolatedScanner(conn.createScanner(mapping.tableName, Authorizations.EMPTY));
+ setFetchColumns(scanner, query.getFields());
+
+ scanner.setRange(createRange(query));
+
+ if (query.getStartTime() != -1 || query.getEndTime() != -1) {
+ IteratorSetting is = new IteratorSetting(30, TimestampFilter.class);
+ if (query.getStartTime() != -1)
+ TimestampFilter.setStart(is, query.getStartTime(), true);
+ if (query.getEndTime() != -1)
+ TimestampFilter.setEnd(is, query.getEndTime(), true);
+
+ scanner.addScanIterator(is);
+ }
+
+ return scanner;
+ }
+
+ /**
+ * Execute the query and return the result.
+ */
+ @Override
+ public Result<K,T> execute(Query<K,T> query) throws GoraException {
+ try {
+ Scanner scanner = createScanner(query);
+ return new IgniteResult<>(this, query, scanner);
+ } catch (TableNotFoundException e) {
+ throw new GoraException(e) ;
+ }
+ }
+
+ @Override
+ public Query<K,T> newQuery() {
+ return new IgniteQuery<>(this);
+ }
+
+ Text pad(Text key, int bytes) {
+ if (key.getLength() < bytes)
+ key = new Text(key);
+
+ while (key.getLength() < bytes) {
+ key.append(new byte[] {0}, 0, 1);
+ }
+
+ return key;
+ }
+
+ @Override
+ public List<PartitionQuery<K,T>> getPartitions(Query<K,T> query) throws GoraException {
+ try {
+ TabletLocator tl;
+ if (conn instanceof MockConnector)
+ tl = new MockTabletLocator();
+ else
+ tl = TabletLocator.getLocator(new ClientContext(conn.getInstance(), credentials, IgniteConfiguration.getTableConfiguration(conn, Tables.getTableId(conn.getInstance(), mapping.tableName))), new Text(Tables.getTableId(conn.getInstance(), mapping.tableName)));
+
+ Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<>();
+
+ tl.invalidateCache();
+ while (tl.binRanges(new ClientContext(conn.getInstance(), credentials, IgniteConfiguration.getTableConfiguration(conn, Tables.getTableId(conn.getInstance(), mapping.tableName))), Collections.singletonList(createRange(query)), binnedRanges).size() > 0) {
+ // TODO log?
+ if (!Tables.exists(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName)))
+ throw new TableDeletedException(Tables.getTableId(conn.getInstance(), mapping.tableName));
+ else if (Tables.getTableState(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName)) == TableState.OFFLINE)
+ throw new TableOfflineException(conn.getInstance(), Tables.getTableId(conn.getInstance(), mapping.tableName));
+ UtilWaitThread.sleep(100);
+ tl.invalidateCache();
+ }
+
+ List<PartitionQuery<K,T>> ret = new ArrayList<>();
+
+ Text startRow = null;
+ Text endRow = null;
+ if (query.getStartKey() != null)
+ startRow = new Text(toBytes(query.getStartKey()));
+ if (query.getEndKey() != null)
+ endRow = new Text(toBytes(query.getEndKey()));
+
+ //hadoop expects hostnames, ignite keeps track of IPs... so need to convert
+ HashMap<String,String> hostNameCache = new HashMap<>();
+
+ for (Entry<String,Map<KeyExtent,List<Range>>> entry : binnedRanges.entrySet()) {
+ String ip = entry.getKey().split(":", 2)[0];
+ String location = hostNameCache.get(ip);
+ if (location == null) {
+ InetAddress inetAddress = InetAddress.getByName(ip);
+ location = inetAddress.getHostName();
+ hostNameCache.put(ip, location);
+ }
+
+ Map<KeyExtent,List<Range>> tablets = entry.getValue();
+ for (KeyExtent ke : tablets.keySet()) {
+
+ K startKey = null;
+ if (startRow == null || !ke.contains(startRow)) {
+ if (ke.getPrevEndRow() != null) {
+ startKey = followingKey(encoder, getKeyClass(), getBytes(ke.getPrevEndRow()));
+ }
+ } else {
+ startKey = fromBytes(getKeyClass(), getBytes(startRow));
+ }
+
+ K endKey = null;
+ if (endRow == null || !ke.contains(endRow)) {
+ if (ke.getEndRow() != null)
+ endKey = lastPossibleKey(encoder, getKeyClass(), getBytes(ke.getEndRow()));
+ } else {
+ endKey = fromBytes(getKeyClass(), getBytes(endRow));
+ }
+
+ PartitionQueryImpl<K, T> pqi = new PartitionQueryImpl<>(query, startKey, endKey, location);
+ pqi.setConf(getConf());
+ ret.add(pqi);
+ }
+ }
+
+ return ret;
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
+
+ }
+
+ static <K> K lastPossibleKey(Encoder encoder, Class<K> clazz, byte[] er) {
+
+ if (clazz.equals(Byte.TYPE) || clazz.equals(Byte.class)) {
+ throw new UnsupportedOperationException();
+ } else if (clazz.equals(Boolean.TYPE) || clazz.equals(Boolean.class)) {
+ throw new UnsupportedOperationException();
+ } else if (clazz.equals(Short.TYPE) || clazz.equals(Short.class)) {
+ return fromBytes(encoder, clazz, encoder.lastPossibleKey(2, er));
+ } else if (clazz.equals(Integer.TYPE) || clazz.equals(Integer.class)) {
+ return fromBytes(encoder, clazz, encoder.lastPossibleKey(4, er));
+ } else if (clazz.equals(Long.TYPE) || clazz.equals(Long.class)) {
+ return fromBytes(encoder, clazz, encoder.lastPossibleKey(8, er));
+ } else if (clazz.equals(Float.TYPE) || clazz.equals(Float.class)) {
+ return fromBytes(encoder, clazz, encoder.lastPossibleKey(4, er));
+ } else if (clazz.equals(Double.TYPE) || clazz.equals(Double.class)) {
+ return fromBytes(encoder, clazz, encoder.lastPossibleKey(8, er));
+ } else if (clazz.equals(String.class)) {
+ throw new UnsupportedOperationException();
+ } else if (clazz.equals(Utf8.class)) {
+ return fromBytes(encoder, clazz, er);
+ }
+
+ throw new IllegalArgumentException(UNKOWN + clazz.getName());
+ }
+
+ @SuppressWarnings("unchecked")
+ static <K> K followingKey(Encoder encoder, Class<K> clazz, byte[] per) {
+
+ if (clazz.equals(Byte.TYPE) || clazz.equals(Byte.class)) {
+ return (K) Byte.valueOf(encoder.followingKey(1, per)[0]);
+ } else if (clazz.equals(Boolean.TYPE) || clazz.equals(Boolean.class)) {
+ throw new UnsupportedOperationException();
+ } else if (clazz.equals(Short.TYPE) || clazz.equals(Short.class)) {
+ return fromBytes(encoder, clazz, encoder.followingKey(2, per));
+ } else if (clazz.equals(Integer.TYPE) || clazz.equals(Integer.class)) {
+ return fromBytes(encoder, clazz, encoder.followingKey(4, per));
+ } else if (clazz.equals(Long.TYPE) || clazz.equals(Long.class)) {
+ return fromBytes(encoder, clazz, encoder.followingKey(8, per));
+ } else if (clazz.equals(Float.TYPE) || clazz.equals(Float.class)) {
+ return fromBytes(encoder, clazz, encoder.followingKey(4, per));
+ } else if (clazz.equals(Double.TYPE) || clazz.equals(Double.class)) {
+ return fromBytes(encoder, clazz, encoder.followingKey(8, per));
+ } else if (clazz.equals(String.class)) {
+ throw new UnsupportedOperationException();
+ } else if (clazz.equals(Utf8.class)) {
+ return fromBytes(encoder, clazz, Arrays.copyOf(per, per.length + 1));
+ }
+
+ throw new IllegalArgumentException(UNKOWN + clazz.getName());
+ }
+
+ @Override
+ public void flush() throws GoraException {
+ try {
+ if (batchWriter != null) {
+ batchWriter.flush();
+ }
+ } catch (Exception e) {
+ throw new GoraException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (batchWriter != null) {
+ batchWriter.close();
+ batchWriter = null;
+ }
+ } catch (MutationsRejectedException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
new file mode 100644
index 0000000..62cba67
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains all the Ignite store related classes.
+ */
+package org.apache.gora.ignite.store;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java b/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java
new file mode 100644
index 0000000..97fb46a
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/util/FixedByteArrayOutputStream.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.util;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * It is a implementation of {@link java.io.OutputStream} must always provide at least a method that writes one byte of output.
+ */
+public class FixedByteArrayOutputStream extends OutputStream {
+
+ private int i;
+ byte out[];
+
+ public FixedByteArrayOutputStream(byte out[]) {
+ this.out = out;
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out[i++] = (byte) b;
+ }
+
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ System.arraycopy(b, off, out, i, len);
+ i += len;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/4346216f/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java
new file mode 100644
index 0000000..eedb84b
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/util/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite store related util classes.
+ */
+package org.apache.gora.ignite.util;
\ No newline at end of file
[11/11] gora git commit: Improve SQL building strategy and add MR
tests for Ignite
Posted by dj...@apache.org.
Improve SQL building strategy and add MR tests for Ignite
The IgniteSQLBuilder class was refactored in in order to simplify the
code. Also, an implementation of DataStoreMapReduceTestBase was added
to the test packages. Finally, some missing formatting issues were
solved.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/446f42ea
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/446f42ea
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/446f42ea
Branch: refs/heads/master
Commit: 446f42ea3685691ee10cf5054a055e8f1685077f
Parents: 8220a15
Author: Carlos M <ca...@gmail.com>
Authored: Sun Sep 9 23:30:53 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Sun Sep 9 23:30:53 2018 -0500
----------------------------------------------------------------------
.gitignore | 1 -
gora-ignite/pom.xml | 9 +-
.../apache/gora/ignite/query/IgniteQuery.java | 5 +
.../apache/gora/ignite/query/IgniteResult.java | 19 +-
.../apache/gora/ignite/query/package-info.java | 4 +-
.../org/apache/gora/ignite/store/Column.java | 32 ++-
.../apache/gora/ignite/store/IgniteMapping.java | 33 +++
.../gora/ignite/store/IgniteMappingBuilder.java | 22 +-
.../gora/ignite/store/IgniteParameters.java | 111 ++++++----
.../apache/gora/ignite/store/IgniteStore.java | 67 +++---
.../apache/gora/ignite/store/package-info.java | 4 +-
.../ignite/utils/IgniteBackendConstants.java | 41 +++-
.../gora/ignite/utils/IgniteSQLBuilder.java | 209 ++++++++++---------
.../apache/gora/ignite/utils/package-info.java | 2 +-
.../mapreduce/IgniteStoreMapReduceTest.java | 65 ++++++
.../gora/ignite/mapreduce/package-info.java | 20 ++
.../org/apache/gora/ignite/package-info.java | 20 ++
.../apache/gora/ignite/store/package-info.java | 20 ++
pom.xml | 117 ++++++-----
19 files changed, 546 insertions(+), 255 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index d4f78bc..2aa54cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,4 +37,3 @@ ivy/ivy*.jar
**/gora-solr-5/src/test/conf/solr/collection1/data/
**/_rest_managed.json
**/gora-solr/src/test/conf/solr-managed-schema/collection1/data/
-**/nbactions.xml
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/pom.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/pom.xml b/gora-ignite/pom.xml
index 8f75553..3e68e3b 100644
--- a/gora-ignite/pom.xml
+++ b/gora-ignite/pom.xml
@@ -110,7 +110,6 @@
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-core</artifactId>
- <version>${ignite.version}</version>
</dependency>
<dependency>
@@ -165,8 +164,12 @@
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-indexing</artifactId>
- <version>${ignite.version}</version>
</dependency>
- </dependencies>
+ <dependency>
+ <groupId>com.healthmarketscience.sqlbuilder</groupId>
+ <artifactId>sqlbuilder</artifactId>
+ </dependency>
+ </dependencies>
+
</project>
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
index 7ad9c61..21715db 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
@@ -19,6 +19,7 @@ package org.apache.gora.ignite.query;
import org.apache.gora.persistency.impl.PersistentBase;
import org.apache.gora.query.impl.QueryBase;
import org.apache.gora.store.DataStore;
+import org.apache.gora.query.Query;
/**
* Ignite specific implementation of the {@link Query} interface.
@@ -28,5 +29,9 @@ public class IgniteQuery<K, T extends PersistentBase> extends QueryBase<K, T> {
public IgniteQuery(DataStore<K, T> dataStore) {
super(dataStore);
}
+
+ public IgniteQuery() {
+ super(null);
+ }
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
index e5c0d46..913e688 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
@@ -35,8 +35,15 @@ public class IgniteResult<K, T extends PersistentBase> extends ResultBase<K, T>
private ResultSet resultSet;
private int size;
- public IgniteResult(DataStore<K, T> dataStore, Query<K, T> query) {
+ public IgniteResult(DataStore<K, T> dataStore, Query<K, T> query, ResultSet resultSet) throws SQLException {
super(dataStore, query);
+ this.resultSet = resultSet;
+ if (resultSet.last()) {
+ size = resultSet.getRow();
+ } else {
+ size = 0;
+ }
+ resultSet.beforeFirst();
}
@Override
@@ -80,14 +87,4 @@ public class IgniteResult<K, T extends PersistentBase> extends ResultBase<K, T>
}
}
- public void setResultSet(ResultSet resultSet) throws SQLException {
- this.resultSet = resultSet;
- if (resultSet.last()) {
- size = resultSet.getRow();
- } else {
- size = 0;
- }
- resultSet.beforeFirst();
- }
-
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
index fa52d9c..0cbd9af 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
@@ -15,6 +15,6 @@
* limitations under the License.
*/
/**
- * This package contains Ignite datastore related all classes.
+ * This package contains Query related classes.
*/
-package org.apache.gora.ignite.query;
\ No newline at end of file
+package org.apache.gora.ignite.query;
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
index ee7d6d6..3034cdf 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
@@ -21,32 +21,56 @@ public class Column {
private String name;
private FieldType dataType;
+ /**
+ * Constructor for Column
+ *
+ * @param name Column's name
+ * @param dataType Column's data type
+ */
public Column(String name, FieldType dataType) {
this.name = name;
this.dataType = dataType;
}
+ /**
+ * Returns the column's name
+ *
+ * @return Column's name
+ */
public String getName() {
return name;
}
+ /**
+ * Sets the column's name
+ *
+ * @param name Column's name
+ */
public void setName(String name) {
this.name = name;
}
+ /**
+ * Returns the column's data-type
+ *
+ * @return Column's data-type
+ */
public FieldType getDataType() {
return dataType;
}
+ /**
+ * Sets the column's data-type
+ *
+ * @param dataType Column's data-type
+ */
public void setDataType(FieldType dataType) {
this.dataType = dataType;
}
-
-
/**
- * For a more detailed list of data types supported by Ignite and its
- * equivalents in Java refer to
+ * Ignite supported data-type enumeration. For a more detailed list of data
+ * types supported by Ignite and its equivalents in Java refer to
* https://apacheignite-sql.readme.io/docs/data-types
*/
public static enum FieldType {
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
index f0189ec..4540ad4 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
@@ -29,30 +29,63 @@ public class IgniteMapping {
private Map<String, Column> fields;
private List<Column> primaryKey;
+ /**
+ * Empty constructor for the IgniteMapping class
+ */
public IgniteMapping() {
fields = new HashMap<>();
}
+ /**
+ * Returns the name of ignite table linked to the mapping.
+ *
+ * @return Table's name.
+ */
public String getTableName() {
return tableName;
}
+ /**
+ * Sets the table name of the ignite mapping
+ *
+ * @param tableName Table's name
+ */
public void setTableName(String tableName) {
this.tableName = tableName;
}
+ /**
+ * Returns a map with all field-column mappings
+ *
+ * @return Map containing mapped fields
+ */
public Map<String, Column> getFields() {
return fields;
}
+ /**
+ * Sets field-column mappings
+ *
+ * @param fields Map containing mapped fields
+ */
public void setFields(Map<String, Column> fields) {
this.fields = fields;
}
+ /**
+ * Returns the primary key's list of columns
+ *
+ * @return List with columns
+ */
public List<Column> getPrimaryKey() {
return primaryKey;
}
+ /**
+ * Sets the primary key's columns
+ *
+ * @param primaryKey List with columns
+ */
public void setPrimaryKey(List<Column> primaryKey) {
this.primaryKey = primaryKey;
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
index 4d397c5..83ad3ba 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
@@ -44,19 +44,39 @@ public class IgniteMappingBuilder<K, T extends PersistentBase> {
private final IgniteStore<K, T> dataStore;
+ /**
+ * Constructor for IgniteMappingBuilder
+ *
+ * @param store IgniteStore instance
+ */
public IgniteMappingBuilder(final IgniteStore<K, T> store) {
this.igniteMapping = new IgniteMapping();
this.dataStore = store;
}
+ /**
+ * Returns the Ignite Mapping being built
+ *
+ * @return Ignite Mapping instance
+ */
public IgniteMapping getIgniteMapping() {
return igniteMapping;
}
+ /**
+ * Sets the Ignite Mapping
+ *
+ * @param igniteMapping Ignite Mapping instance
+ */
public void setIgniteMapping(IgniteMapping igniteMapping) {
this.igniteMapping = igniteMapping;
}
+ /**
+ * Reads Ignite mappings from file
+ *
+ * @param mappingFile File name relative to the resource's classpath
+ */
public void readMappingFile(String mappingFile) {
try {
SAXBuilder saxBuilder = new SAXBuilder();
@@ -102,11 +122,9 @@ public class IgniteMappingBuilder<K, T extends PersistentBase> {
break;
}
}
-
} catch (IOException | JDOMException | ConfigurationException e) {
throw new RuntimeException(e);
}
LOG.info("Gora Ignite mapping file was read successfully.");
-
}
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
index f667b40..6606583 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
@@ -18,43 +18,12 @@ package org.apache.gora.ignite.store;
import java.util.Properties;
import org.apache.gora.ignite.utils.IgniteBackendConstants;
-import org.apache.hadoop.conf.Configuration;
/**
* Parameters definitions for Ignite.
*/
public class IgniteParameters {
- /**
- * Property indicating the Ignite Schema to be used
- */
- public static final String PROP_SCHEMA = "gora.datastore.ignite.schema";
-
- /**
- * Property indicating the Ignite Cluster Node IP
- */
- public static final String PROP_HOST = "gora.datastore.ignite.host";
-
- /**
- * Property indicating the port used by the Ignite Server
- */
- public static final String PROP_PORT = "gora.datastore.ignite.port";
-
- /**
- * Property indicating the username to connect to the server
- */
- public static final String PROP_USER = "gora.datastore.ignite.user";
-
- /**
- * Property indicating the password to connect to the server
- */
- public static final String PROP_PASSWORD = "gora.datastore.ignite.password";
-
- /**
- * Property indicating additional JDBC options
- */
- public static final String PROP_ADDITIONALS = "gora.datastore.ignite.additionalConfigurations";
-
private String host;
private String port;
private String schema;
@@ -79,62 +48,130 @@ public class IgniteParameters {
this.additionalConfigurations = additionalConfigurations;
}
+ /**
+ * Returns the ignite hostname
+ *
+ * @return IP/domain of the ignite server
+ */
public String getHost() {
return host;
}
+ /**
+ * Sets the ignite hostname
+ *
+ * @param host IP/domain of the ignite server
+ */
public void setHost(String host) {
this.host = host;
}
+ /**
+ * Returns the ignite port
+ *
+ * @return Port of the ignite server eg. 10800
+ */
public String getPort() {
return port;
}
+ /**
+ * Sets the ignite port
+ *
+ * @param port Port of the ignite server eg. 10800
+ */
public void setPort(String port) {
this.port = port;
}
+ /**
+ * Returns the username used for the ignite connection
+ *
+ * @return Username of ignite
+ */
public String getUser() {
return user;
}
+ /**
+ * Sets the username used for the ignite connection
+ *
+ * @param user Username of ignite
+ */
public void setUser(String user) {
this.user = user;
}
+ /**
+ * Returns the secrets used for the ignite connection
+ *
+ * @return Password of the ignite user
+ */
public String getPassword() {
return password;
}
+ /**
+ * Sets the secrets used for the ignite connection
+ *
+ * @param password Password of the ignite user
+ */
public void setPassword(String password) {
this.password = password;
}
+ /**
+ * Returns additional configurations used for the JDBC connection For more
+ * details refer to https://apacheignite-sql.readme.io/docs/jdbc-driver
+ *
+ * @return String containing JDBC configurations
+ */
public String getAdditionalConfigurations() {
return additionalConfigurations;
}
+ /**
+ * Sets additional configurations used for the JDBC connection. For more
+ * details refer to https://apacheignite-sql.readme.io/docs/jdbc-driver
+ *
+ * @param additionalConfigurations String containing JDBC configurations
+ */
public void setAdditionalConfigurations(String additionalConfigurations) {
this.additionalConfigurations = additionalConfigurations;
}
+ /**
+ * Returns the ignite schema for the JDBC connection
+ *
+ * @return Ignite schema e.g. PUBLIC
+ */
public String getSchema() {
return schema;
}
+ /**
+ * Sets the ignite schema for the JDBC connection
+ *
+ * @param schema Ignite schema e.g. PUBLIC
+ */
public void setSchema(String schema) {
this.schema = schema;
}
+ /**
+ * Reads Ignite parameters from a properties list
+ *
+ * @param properties Properties list
+ * @return Ignite parameters instance
+ */
public static IgniteParameters load(Properties properties) {
return new IgniteParameters(
- properties.getProperty(PROP_HOST, IgniteBackendConstants.DEFAULT_IGNITE_HOST),
- properties.getProperty(PROP_PORT, IgniteBackendConstants.DEFAULT_IGNITE_PORT),
- properties.getProperty(PROP_SCHEMA, null),
- properties.getProperty(PROP_USER, null),
- properties.getProperty(PROP_PASSWORD, null),
- properties.getProperty(PROP_ADDITIONALS, null));
+ properties.getProperty(IgniteBackendConstants.PROP_HOST, IgniteBackendConstants.DEFAULT_IGNITE_HOST),
+ properties.getProperty(IgniteBackendConstants.PROP_PORT, IgniteBackendConstants.DEFAULT_IGNITE_PORT),
+ properties.getProperty(IgniteBackendConstants.PROP_SCHEMA),
+ properties.getProperty(IgniteBackendConstants.PROP_USER),
+ properties.getProperty(IgniteBackendConstants.PROP_PASSWORD),
+ properties.getProperty(IgniteBackendConstants.PROP_ADDITIONALS));
}
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
index a17b3a0..4f1abc9 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -81,6 +81,9 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
igniteParameters = IgniteParameters.load(properties);
connection = acquireConnection();
LOG.info("Ignite store was successfully initialized");
+ if (!schemaExists()) {
+ createSchema();
+ }
} catch (ClassNotFoundException | SQLException ex) {
LOG.error("Error while initializing Ignite store", ex);
throw new GoraException(ex);
@@ -88,9 +91,9 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
}
private Connection acquireConnection() throws ClassNotFoundException, SQLException {
- Class.forName("org.apache.ignite.IgniteJdbcThinDriver");
+ Class.forName(IgniteBackendConstants.DRIVER_NAME);
StringBuilder urlBuilder = new StringBuilder();
- urlBuilder.append("jdbc:ignite:thin://");
+ urlBuilder.append(IgniteBackendConstants.JDBC_PREFIX);
urlBuilder.append(igniteParameters.getHost());
if (igniteParameters.getPort() != null) {
urlBuilder.append(":" + igniteParameters.getPort());
@@ -187,9 +190,9 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
for (String af : avFields) {
dbFields.add(igniteMapping.getFields().get(af).getName());
}
- String selectQuery = IgniteSQLBuilder.selectGet(igniteMapping, dbFields);
+ String selectQuery = IgniteSQLBuilder.createSelectQueryGet(igniteMapping, dbFields);
try (PreparedStatement stmt = connection.prepareStatement(selectQuery)) {
- IgniteSQLBuilder.fillSelectStatement(stmt, igniteMapping, keyl);
+ IgniteSQLBuilder.fillSelectQuery(stmt, igniteMapping, keyl);
ResultSet rs = stmt.executeQuery();
boolean data = rs.next();
T resp = null;
@@ -204,21 +207,20 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
} catch (SQLException | IOException ex) {
throw new GoraException(ex);
}
-
}
- public T newInstance(ResultSet rs, String[] fields) throws GoraException, SQLException, IOException {
+ public T newInstance(ResultSet resultset, String[] fields) throws GoraException, SQLException, IOException {
fields = getFieldsToQuery(fields);
T persistent = newPersistent();
for (String f : fields) {
Schema.Field field = fieldMap.get(f);
Schema fieldSchema = field.schema();
String dbField = igniteMapping.getFields().get(f).getName();
- Object sv = rs.getObject(dbField);
- if (sv == null) {
+ Object fieldValue = resultset.getObject(dbField);
+ if (fieldValue == null) {
continue;
}
- Object v = deserializeFieldValue(field, fieldSchema, sv, persistent);
+ Object v = deserializeFieldValue(field, fieldSchema, fieldValue, persistent);
persistent.put(field.pos(), v);
persistent.setDirty(field.pos());
}
@@ -263,7 +265,6 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
fieldValue = igniteValue;
}
return fieldValue;
-
}
@Override
@@ -280,17 +281,17 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
//Composite keys pending..
}
for (Schema.Field field : fields) {
- Column get = igniteMapping.getFields().get(field.name());
- Object v = obj.get(field.pos());
- if (get != null && v != null) {
+ Column mappedColumn = igniteMapping.getFields().get(field.name());
+ Object fieldValue = obj.get(field.pos());
+ if (mappedColumn != null && fieldValue != null) {
Schema fieldSchema = field.schema();
- Object serializedObj = serializeFieldValue(fieldSchema, v);
- data.put(get, serializedObj);
+ Object serializedObj = serializeFieldValue(fieldSchema, fieldValue);
+ data.put(mappedColumn, serializedObj);
}
}
- String baseInsertStatement = IgniteSQLBuilder.baseInsertStatement(igniteMapping, data);
+ String baseInsertStatement = IgniteSQLBuilder.createInsertQuery(igniteMapping, data);
try (PreparedStatement stmt = connection.prepareStatement(baseInsertStatement)) {
- IgniteSQLBuilder.fillInsertStatement(stmt, data);
+ IgniteSQLBuilder.fillInsertQuery(stmt, data);
stmt.executeUpdate();
} catch (SQLException ex) {
throw new GoraException(ex);
@@ -307,15 +308,15 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public boolean delete(K key) throws GoraException {
String deleteQuery = null;
- Object[] keyl = null;
+ Object[] keyArray = null;
if (igniteMapping.getPrimaryKey().size() == 1) {
- deleteQuery = IgniteSQLBuilder.delete(igniteMapping);
- keyl = new Object[]{key};
+ deleteQuery = IgniteSQLBuilder.createDeleteQuery(igniteMapping);
+ keyArray = new Object[]{key};
} else {
//Composite key pending
}
try (PreparedStatement stmt = connection.prepareStatement(deleteQuery)) {
- IgniteSQLBuilder.fillDeleteStatement(stmt, igniteMapping, keyl);
+ IgniteSQLBuilder.fillDeleteQuery(stmt, igniteMapping, keyArray);
stmt.executeUpdate();
return true;
} catch (SQLException ex) {
@@ -325,20 +326,19 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public long deleteByQuery(Query<K, T> query) throws GoraException {
-
String deleteQuery;
if (query.getFields() != null && query.getFields().length < igniteMapping.getFields().size()) {
List<String> dbFields = new ArrayList<>();
for (String af : query.getFields()) {
dbFields.add(igniteMapping.getFields().get(af).getName());
}
- deleteQuery = IgniteSQLBuilder.deleteQueryFields(igniteMapping, dbFields);
+ deleteQuery = IgniteSQLBuilder.createDeleteQueryWithFields(igniteMapping, dbFields);
} else {
- deleteQuery = IgniteSQLBuilder.deleteQuery(igniteMapping);
+ deleteQuery = IgniteSQLBuilder.createDeleteQueryMultipleRecords(igniteMapping);
}
- String selectQueryWhere = IgniteSQLBuilder.queryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
+ String selectQueryWhere = IgniteSQLBuilder.createWhereClause(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
try (PreparedStatement stmt = connection.prepareStatement(deleteQuery + selectQueryWhere)) {
- IgniteSQLBuilder.fillSelectQuery(stmt, query.getStartKey(), query.getEndKey());
+ IgniteSQLBuilder.fillWhereClause(stmt, query.getStartKey(), query.getEndKey());
stmt.executeUpdate();
return 0;
} catch (SQLException ex) {
@@ -351,19 +351,18 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
String[] fields = getFieldsToQuery(query.getFields());
//Avro fields to Ignite fields
List<String> dbFields = new ArrayList<>();
- for (String af : fields) {
- dbFields.add(igniteMapping.getFields().get(af).getName());
+ for (String aField : fields) {
+ dbFields.add(igniteMapping.getFields().get(aField).getName());
}
- String selectQuery = IgniteSQLBuilder.selectQuery(igniteMapping, dbFields);
- String selectQueryWhere = IgniteSQLBuilder.queryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
+ String selectQuery = IgniteSQLBuilder.createSelectQuery(igniteMapping, dbFields);
+ String selectQueryWhere = IgniteSQLBuilder.createWhereClause(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
try (PreparedStatement stmt = connection.prepareStatement(selectQuery + selectQueryWhere)) {
RowSetFactory factory = RowSetProvider.newFactory();
CachedRowSet rowset = factory.createCachedRowSet();
- IgniteSQLBuilder.fillSelectQuery(stmt, query.getStartKey(), query.getEndKey());
+ IgniteSQLBuilder.fillWhereClause(stmt, query.getStartKey(), query.getEndKey());
ResultSet executeQuery = stmt.executeQuery();
rowset.populate(executeQuery);
- IgniteResult<K, T> igniteResult = new IgniteResult<>(this, query);
- igniteResult.setResultSet(rowset);
+ IgniteResult<K, T> igniteResult = new IgniteResult<>(this, query, rowset);
return igniteResult;
} catch (SQLException ex) {
throw new GoraException(ex);
@@ -462,7 +461,6 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
break;
default:
throw new AssertionError(fieldSchema.getType().name());
-
}
return output;
}
@@ -558,7 +556,6 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
writer = new SpecificDatumWriter(fieldSchema);// ignore dirty bits
writerMap.put(fieldSchema, writer);
}
-
return writer;
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
index 49959c8..e747477 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
@@ -15,6 +15,6 @@
* limitations under the License.
*/
/**
- * This package contains all the Ignite store related classes.
+ * This package contains the core classes of the ignite datastore.
*/
-package org.apache.gora.ignite.store;
\ No newline at end of file
+package org.apache.gora.ignite.store;
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
index 7459c67..9baa131 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
@@ -22,15 +22,52 @@ package org.apache.gora.ignite.utils;
*/
public class IgniteBackendConstants {
+ /**
+ * Ignite JDBC constants
+ */
+ public static final String DRIVER_NAME = "org.apache.ignite.IgniteJdbcThinDriver";
+ public static final String JDBC_PREFIX = "jdbc:ignite:thin://";
+
/*
* Default configurations for Ignite
- */
+ */
public static final String DEFAULT_IGNITE_HOST = "localhost";
public static final String DEFAULT_IGNITE_PORT = "10800";
-
+
/*
* A '42000' error code is thrown by Ignite when a non-existent table is queried.
* More details: https://apacheignite-sql.readme.io/docs/jdbc-error-codes
*/
public static final String DEFAULT_IGNITE_TABLE_NOT_EXISTS_CODE = "42000";
+
+ /**
+ * Property indicating the Ignite Schema to be used
+ */
+ public static final String PROP_SCHEMA = "gora.datastore.ignite.schema";
+
+ /**
+ * Property indicating the Ignite Cluster Node IP
+ */
+ public static final String PROP_HOST = "gora.datastore.ignite.host";
+
+ /**
+ * Property indicating the port used by the Ignite Server
+ */
+ public static final String PROP_PORT = "gora.datastore.ignite.port";
+
+ /**
+ * Property indicating the username to connect to the server
+ */
+ public static final String PROP_USER = "gora.datastore.ignite.user";
+
+ /**
+ * Property indicating the password to connect to the server
+ */
+ public static final String PROP_PASSWORD = "gora.datastore.ignite.password";
+
+ /**
+ * Property indicating additional JDBC options
+ */
+ public static final String PROP_ADDITIONALS = "gora.datastore.ignite.additionalConfigurations";
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
index 3639b90..85c2960 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
@@ -17,12 +17,26 @@
package org.apache.gora.ignite.utils;
import avro.shaded.com.google.common.collect.Lists;
+import com.healthmarketscience.sqlbuilder.BinaryCondition;
+import com.healthmarketscience.sqlbuilder.CreateTableQuery;
+import com.healthmarketscience.sqlbuilder.CustomSql;
+import com.healthmarketscience.sqlbuilder.DeleteQuery;
+import com.healthmarketscience.sqlbuilder.DropQuery;
+import com.healthmarketscience.sqlbuilder.InsertQuery;
+import com.healthmarketscience.sqlbuilder.SelectQuery;
+import com.healthmarketscience.sqlbuilder.SqlObject;
+import com.healthmarketscience.sqlbuilder.UpdateQuery;
+import com.healthmarketscience.sqlbuilder.custom.mysql.MysLimitClause;
+import com.healthmarketscience.sqlbuilder.dbspec.Constraint;
+import com.healthmarketscience.sqlbuilder.dbspec.basic.DbColumn;
+import com.healthmarketscience.sqlbuilder.dbspec.basic.DbConstraint;
+import com.healthmarketscience.sqlbuilder.dbspec.basic.DbSchema;
+import com.healthmarketscience.sqlbuilder.dbspec.basic.DbSpec;
+import com.healthmarketscience.sqlbuilder.dbspec.basic.DbTable;
import java.sql.PreparedStatement;
import java.sql.SQLException;
-import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.gora.ignite.store.Column;
@@ -34,11 +48,6 @@ import org.apache.gora.ignite.store.IgniteMapping;
*/
public class IgniteSQLBuilder {
- private static String format(String pattern, Object... args) {
- MessageFormat messageFormat = new MessageFormat(pattern, Locale.getDefault());
- return messageFormat.format(args);
- }
-
/**
* Returns a SQL query for determine whether a table exists or not.
*
@@ -46,7 +55,11 @@ public class IgniteSQLBuilder {
* @return SQL query
*/
public static String tableExists(String tableName) {
- return format("SELECT * FROM {0} LIMIT 0", tableName);
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(tableName);
+ return new SelectQuery().addAllColumns().addFromTable(aTable)
+ .addCustomization(new MysLimitClause(0)).validate().toString();
}
/**
@@ -57,26 +70,24 @@ public class IgniteSQLBuilder {
* @return SQL create query (DDL).
*/
public static String createTable(IgniteMapping mapping) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("CREATE TABLE ");
- sqlBuilder.append(mapping.getTableName());
- sqlBuilder.append("(");
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
ArrayList<Column> fieldsList = Lists.newArrayList(mapping.getPrimaryKey());
fieldsList.addAll(Lists.newArrayList(mapping.getFields().values()));
for (Column aColumn : fieldsList) {
String name = aColumn.getName();
Column.FieldType dataType = aColumn.getDataType();
- sqlBuilder.append(name).append(" ").append(dataType.toString()).append(",");
+ aTable.addColumn(name, dataType.toString(), null);
}
- sqlBuilder.append("PRIMARY KEY ");
- sqlBuilder.append("(");
+ String[] keys = new String[mapping.getPrimaryKey().size()];
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
- sqlBuilder.append(mapping.getPrimaryKey().get(i).getName());
- sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : ",");
+ keys[i] = mapping.getPrimaryKey().get(i).getName();
}
- sqlBuilder.append(")");
- sqlBuilder.append(");");
- return sqlBuilder.toString();
+ aTable.addConstraint(new DbConstraint(aTable,
+ mapping.getTableName() + "_PRIMARY_KEY",
+ Constraint.Type.PRIMARY_KEY, keys));
+ return new CreateTableQuery(aTable, true).validate().toString();
}
/**
@@ -87,7 +98,8 @@ public class IgniteSQLBuilder {
* @return SQL drop query (DDL).
*/
public static String dropTable(String tableName) {
- return format("DROP TABLE IF EXISTS {0} ;", tableName);
+ String statement = DropQuery.dropTable(tableName).validate().toString();
+ return statement.substring(0, 11) + "IF EXISTS " + statement.substring(11);
}
/**
@@ -98,25 +110,18 @@ public class IgniteSQLBuilder {
* @param data A map containing the Column-Value pairs of the new record.
* @return SQL insert statement
*/
- public static String baseInsertStatement(IgniteMapping mapping, Map<Column, Object> data) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("MERGE INTO ");
- sqlBuilder.append(mapping.getTableName());
- sqlBuilder.append(" (");
+ public static String createInsertQuery(IgniteMapping mapping, Map<Column, Object> data) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ InsertQuery insertQuery = new InsertQuery(aTable);
List<Entry<Column, Object>> list = new ArrayList<>(data.entrySet());
+ String[] columns = new String[list.size()];
for (int i = 0; i < list.size(); i++) {
- sqlBuilder.append(list.get(i).getKey().getName());
- sqlBuilder.append(i == list.size() - 1 ? "" : ",");
- }
- sqlBuilder.append(")");
- sqlBuilder.append(" VALUES ");
- sqlBuilder.append(" (");
- for (int i = 0; i < list.size(); i++) {
- sqlBuilder.append("?");
- sqlBuilder.append(i == list.size() - 1 ? "" : ",");
+ columns[i] = list.get(i).getKey().getName();
}
- sqlBuilder.append(" )");
- return sqlBuilder.toString();
+ return insertQuery.addCustomPreparedColumns(columns).validate().toString()
+ .replaceFirst("INSERT", "MERGE");
}
/**
@@ -129,7 +134,7 @@ public class IgniteSQLBuilder {
* @throws SQLException When invalid values are provided as parameters for the
* insert statement.
*/
- public static void fillInsertStatement(PreparedStatement statement, Map<Column, Object> insertData) throws SQLException {
+ public static void fillInsertQuery(PreparedStatement statement, Map<Column, Object> insertData) throws SQLException {
List<Entry<Column, Object>> list = new ArrayList<>(insertData.entrySet());
for (int i = 0; i < list.size(); i++) {
int j = i + 1;
@@ -144,17 +149,17 @@ public class IgniteSQLBuilder {
* @param mapping The ignite mapping definition of the data store
* @return SQL delete statement
*/
- public static String delete(IgniteMapping mapping) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("DELETE FROM ");
- sqlBuilder.append(mapping.getTableName());
- sqlBuilder.append(" WHERE ");
+ public static String createDeleteQuery(IgniteMapping mapping) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ DeleteQuery statement = new DeleteQuery(aTable);
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
- sqlBuilder.append(mapping.getPrimaryKey().get(i).getName());
- sqlBuilder.append("= ? ");
- sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : " AND ");
+ statement.addCondition(new BinaryCondition(BinaryCondition.Op.EQUAL_TO,
+ new DbColumn(aTable, mapping.getPrimaryKey().get(i).getName(), null),
+ SqlObject.QUESTION_MARK));
}
- return sqlBuilder.toString();
+ return statement.validate().toString();
}
/**
@@ -167,7 +172,7 @@ public class IgniteSQLBuilder {
* record to be deleted
* @throws SQLException When invalid keys' values are provided as parameters
*/
- public static void fillDeleteStatement(PreparedStatement statement, IgniteMapping mapping, Object... deleteData) throws SQLException {
+ public static void fillDeleteQuery(PreparedStatement statement, IgniteMapping mapping, Object... deleteData) throws SQLException {
assert mapping.getPrimaryKey().size() == deleteData.length;
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
int j = i + 1;
@@ -183,22 +188,23 @@ public class IgniteSQLBuilder {
* @param columns A list of columns to be retrieved within the select query
* @return SQL select statement
*/
- public static String selectGet(IgniteMapping mapping, List<String> columns) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("SELECT ");
+ public static String createSelectQueryGet(IgniteMapping mapping, List<String> columns) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ SelectQuery selectQuery = new SelectQuery();
+ selectQuery.addFromTable(aTable);
+ DbColumn[] lsColumns = new DbColumn[columns.size()];
for (int i = 0; i < columns.size(); i++) {
- sqlBuilder.append(columns.get(i));
- sqlBuilder.append(i == columns.size() - 1 ? "" : " , ");
+ lsColumns[i] = aTable.addColumn(columns.get(i));
}
- sqlBuilder.append(" FROM ");
- sqlBuilder.append(mapping.getTableName());
- sqlBuilder.append(" WHERE ");
+ selectQuery.addColumns(lsColumns);
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
- sqlBuilder.append(mapping.getPrimaryKey().get(i).getName());
- sqlBuilder.append("= ? ");
- sqlBuilder.append(i == mapping.getPrimaryKey().size() - 1 ? "" : " AND ");
+ selectQuery.addCondition(new BinaryCondition(BinaryCondition.Op.EQUAL_TO,
+ new DbColumn(aTable, mapping.getPrimaryKey().get(i).getName(), null),
+ SqlObject.QUESTION_MARK));
}
- return sqlBuilder.toString();
+ return selectQuery.validate().toString();
}
/**
@@ -211,7 +217,7 @@ public class IgniteSQLBuilder {
* record to be retrieved
* @throws SQLException When invalid keys' values are provided as parameters
*/
- public static void fillSelectStatement(PreparedStatement statement, IgniteMapping mapping, Object... selectData) throws SQLException {
+ public static void fillSelectQuery(PreparedStatement statement, IgniteMapping mapping, Object... selectData) throws SQLException {
assert mapping.getPrimaryKey().size() == selectData.length;
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
int j = i + 1;
@@ -228,21 +234,23 @@ public class IgniteSQLBuilder {
* query
* @return SQL select statement
*/
- public static String selectQuery(IgniteMapping mapping, List<String> selectFields) {
+ public static String createSelectQuery(IgniteMapping mapping, List<String> selectFields) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ SelectQuery selectQuery = new SelectQuery();
+ selectQuery.addFromTable(aTable);
List<String> fields = new ArrayList<>();
for (Column c : mapping.getPrimaryKey()) {
fields.add(c.getName());
}
fields.addAll(selectFields);
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("SELECT ");
+ DbColumn[] lsColumns = new DbColumn[fields.size()];
for (int i = 0; i < fields.size(); i++) {
- sqlBuilder.append(fields.get(i));
- sqlBuilder.append(i == fields.size() - 1 ? "" : " , ");
+ lsColumns[i] = aTable.addColumn(fields.get(i));
}
- sqlBuilder.append(" FROM ");
- sqlBuilder.append(mapping.getTableName());
- return sqlBuilder.toString();
+ selectQuery.addColumns(lsColumns);
+ return selectQuery.validate().toString();
}
/**
@@ -252,11 +260,12 @@ public class IgniteSQLBuilder {
* @param mapping The ignite mapping definition of the data store
* @return SQL delete statement
*/
- public static String deleteQuery(IgniteMapping mapping) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("DELETE FROM ");
- sqlBuilder.append(mapping.getTableName());
- return sqlBuilder.toString();
+ public static String createDeleteQueryMultipleRecords(IgniteMapping mapping) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ DeleteQuery deleteQuery = new DeleteQuery(aTable);
+ return deleteQuery.validate().toString();
}
/**
@@ -267,19 +276,15 @@ public class IgniteSQLBuilder {
* @param deleteFields A list of columns to be deleted (set to null)
* @return SQL update statement
*/
- public static String deleteQueryFields(IgniteMapping mapping, List<String> deleteFields) {
- StringBuilder sqlBuilder = new StringBuilder();
- sqlBuilder.append("UPDATE ");
- sqlBuilder.append(mapping.getTableName());
- if (!deleteFields.isEmpty()) {
- sqlBuilder.append(" SET ");
- }
+ public static String createDeleteQueryWithFields(IgniteMapping mapping, List<String> deleteFields) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ UpdateQuery updateQuery = new UpdateQuery(aTable);
for (int i = 0; i < deleteFields.size(); i++) {
- sqlBuilder.append(deleteFields.get(i));
- sqlBuilder.append(" = null");
- sqlBuilder.append(i == deleteFields.size() - 1 ? "" : " , ");
+ updateQuery.addSetClause(new DbColumn(aTable, deleteFields.get(i), null), SqlObject.NULL_VALUE);
}
- return sqlBuilder.toString();
+ return updateQuery.validate().toString();
}
/**
@@ -292,34 +297,34 @@ public class IgniteSQLBuilder {
* @param limit The maximum number of records to be consider
* @return SQL WHERE segment
*/
- public static String queryWhere(IgniteMapping mapping, Object startKey, Object endKey, long limit) {
- //composite keys pending
- assert mapping.getPrimaryKey().size() == 1;
+ public static String createWhereClause(IgniteMapping mapping, Object startKey, Object endKey, long limit) {
+ DbSpec spec = new DbSpec();
+ DbSchema schema = spec.addDefaultSchema();
+ DbTable aTable = schema.addTable(mapping.getTableName());
+ SelectQuery selectQuery = new SelectQuery();
+ selectQuery.addFromTable(aTable);
+ String fisrtPart = selectQuery.validate().toString();
String keycolumn = mapping.getPrimaryKey().get(0).getName();
- StringBuilder sqlBuilder = new StringBuilder();
if (startKey != null || endKey != null) {
- sqlBuilder.append(" WHERE ");
if (startKey != null && endKey != null && startKey.equals(endKey)) {
- sqlBuilder.append(keycolumn);
- sqlBuilder.append("= ?");
+ selectQuery.addCondition(new BinaryCondition(BinaryCondition.Op.EQUAL_TO,
+ new CustomSql(keycolumn), SqlObject.QUESTION_MARK));
} else {
if (startKey != null) {
- sqlBuilder.append(keycolumn);
- sqlBuilder.append(">= ?");
- }
- if (startKey != null && endKey != null) {
- sqlBuilder.append(" AND ");
+ selectQuery.addCondition(new BinaryCondition(BinaryCondition.Op.GREATER_THAN_OR_EQUAL_TO,
+ new CustomSql(keycolumn), SqlObject.QUESTION_MARK));
}
if (endKey != null) {
- sqlBuilder.append(keycolumn);
- sqlBuilder.append("<= ?");
+ selectQuery.addCondition(new BinaryCondition(BinaryCondition.Op.LESS_THAN_OR_EQUAL_TO,
+ new CustomSql(keycolumn), SqlObject.QUESTION_MARK));
}
}
}
if (limit > 0) {
- sqlBuilder.append(" LIMIT ").append(limit);
+ selectQuery.addCustomization(new MysLimitClause(limit));
}
- return sqlBuilder.toString();
+ String completeQuery = selectQuery.validate().toString();
+ return completeQuery.substring(fisrtPart.length());
}
/**
@@ -331,7 +336,7 @@ public class IgniteSQLBuilder {
* @param endKey End key of the WHERE condition
* @throws SQLException When invalid keys' values are provided as parameters
*/
- public static void fillSelectQuery(PreparedStatement statement, Object startKey, Object endKey) throws SQLException {
+ public static void fillWhereClause(PreparedStatement statement, Object startKey, Object endKey) throws SQLException {
if (startKey != null || endKey != null) {
if (startKey != null && endKey != null && startKey.equals(endKey)) {
statement.setObject(1, startKey);
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
index ddba6e6..78fa1d0 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
@@ -15,6 +15,6 @@
* limitations under the License.
*/
/**
- * This package contains Ignite datastore related all classes.
+ * This package contains utility classes.
*/
package org.apache.gora.ignite.utils;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/IgniteStoreMapReduceTest.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/IgniteStoreMapReduceTest.java b/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/IgniteStoreMapReduceTest.java
new file mode 100644
index 0000000..2f82247
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/IgniteStoreMapReduceTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.mapreduce;
+
+import java.io.IOException;
+import org.apache.gora.examples.generated.WebPage;
+import org.apache.gora.ignite.GoraIgniteTestDriver;
+import org.apache.gora.mapreduce.DataStoreMapReduceTestBase;
+import org.apache.gora.store.DataStore;
+import org.apache.gora.store.DataStoreFactory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Executes tests for MR jobs over Ignite dataStore.
+ */
+public class IgniteStoreMapReduceTest extends DataStoreMapReduceTestBase {
+
+ private GoraIgniteTestDriver driver;
+
+ public IgniteStoreMapReduceTest() throws IOException {
+ super();
+ driver = new GoraIgniteTestDriver();
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ driver.setUpClass();
+ super.setUp();
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ driver.tearDownClass();
+ }
+
+ @Override
+ protected DataStore<String, WebPage> createWebPageDataStore() throws IOException {
+ try {
+ return DataStoreFactory.getDataStore(String.class, WebPage.class, driver.getConfiguration());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/package-info.java
new file mode 100644
index 0000000..6dd4be7
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/mapreduce/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains map reduce tests.
+ */
+package org.apache.gora.ignite.mapreduce;
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
new file mode 100644
index 0000000..1899e5d
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite datastore test utilities.
+ */
+package org.apache.gora.ignite;
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java b/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
new file mode 100644
index 0000000..03fc7e3
--- /dev/null
+++ b/gora-ignite/src/test/java/org/apache/gora/ignite/store/package-info.java
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite datastore tests.
+ */
+package org.apache.gora.ignite.store;
http://git-wip-us.apache.org/repos/asf/gora/blob/446f42ea/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index de30ec6..ddd0d14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -433,7 +433,7 @@
</dependencies>
</plugin>
<!--This plugin's configuration is used to store Eclipse m2e settings
- only. It has no influence on the Maven build itself. -->
+ only. It has no influence on the Maven build itself. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
@@ -647,17 +647,17 @@
<build>
<plugins>
<!-- <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version> <executions> <execution>
- <id>rat-verify</id> <phase>test</phase> <goals> <goal>check</goal> </goals>
- </execution> </executions> <configuration> <licenses> <license implementation="org.apache.rat.analysis.license.SimplePatternBasedLicense">
- <licenseFamilyCategory>ASL20</licenseFamilyCategory> <licenseFamilyName>Apache
- Software License, 2.0</licenseFamilyName> <notes>Single licensed ASL v2.0</notes>
- <patterns> <pattern>Licensed to the Apache Software Foundation (ASF) under
- one or more contributor license agreements.</pattern> </patterns> </license>
- </licenses> <excludeSubProjects>false</excludeSubProjects> <excludes> <exclude>CHANGES.txt</exclude>
- <exclude>README.txt</exclude> <exclude>NOTICE.txt</exclude> <exclude>LICENSE.txt</exclude>
- <exclude>KEYS</exclude> <exclude>doap_Gora.rdf</exclude> <exclude>.gitignore/**/**</exclude>
- </excludes> </configuration> </plugin -->
+ <version>${apache-rat-plugin.version}</version> <executions> <execution>
+ <id>rat-verify</id> <phase>test</phase> <goals> <goal>check</goal> </goals>
+ </execution> </executions> <configuration> <licenses> <license implementation="org.apache.rat.analysis.license.SimplePatternBasedLicense">
+ <licenseFamilyCategory>ASL20</licenseFamilyCategory> <licenseFamilyName>Apache
+ Software License, 2.0</licenseFamilyName> <notes>Single licensed ASL v2.0</notes>
+ <patterns> <pattern>Licensed to the Apache Software Foundation (ASF) under
+ one or more contributor license agreements.</pattern> </patterns> </license>
+ </licenses> <excludeSubProjects>false</excludeSubProjects> <excludes> <exclude>CHANGES.txt</exclude>
+ <exclude>README.txt</exclude> <exclude>NOTICE.txt</exclude> <exclude>LICENSE.txt</exclude>
+ <exclude>KEYS</exclude> <exclude>doap_Gora.rdf</exclude> <exclude>.gitignore/**/**</exclude>
+ </excludes> </configuration> </plugin -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
@@ -768,7 +768,7 @@
<osgi.version>4.2.0</osgi.version>
<!-- Avro Dependencies -->
<avro.version>1.8.1</avro.version>
-<!-- <jackson.version>1.6.9</jackson.version> -->
+ <!-- <jackson.version>1.6.9</jackson.version> -->
<!-- Hadoop Dependencies -->
<hadoop-2.version>2.5.2</hadoop-2.version>
<hadoop-2.test.version>2.5.2</hadoop-2.test.version>
@@ -782,6 +782,7 @@
<cassandra.version>3.11.0</cassandra.version>
<!-- Ignite Dependencies -->
<ignite.version>2.6.0</ignite.version>
+ <sqlbuilder.version>2.1.7</sqlbuilder.version>
<!-- Solr Dependencies -->
<lucene-solr.version>6.5.1</lucene-solr.version>
<solr-solrj.version>6.5.1</solr-solrj.version>
@@ -970,6 +971,11 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>com.healthmarketscience.sqlbuilder</groupId>
+ <artifactId>sqlbuilder</artifactId>
+ <version>${sqlbuilder.version}</version>
+ </dependency>
<!-- End of Internal Depednecies -->
@@ -1045,26 +1051,26 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>avro</artifactId>
</exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-auth</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -1077,26 +1083,26 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>avro</artifactId>
</exclusion>
- <exclusion>
- <artifactId>slf4j-log4j12</artifactId>
- <groupId>org.slf4j</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-yarn-common</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
- <exclusion>
- <artifactId>hadoop-auth</artifactId>
- <groupId>org.apache.hadoop</groupId>
- </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ </exclusion>
</exclusions>
</dependency>
@@ -1595,6 +1601,11 @@
<artifactId>ignite-indexing</artifactId>
<version>${ignite.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${ignite.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
[08/11] gora git commit: Implement Queries for Ignite
Posted by dj...@apache.org.
Implement Queries for Ignite
The operations newQuery, execute, deleteByQuery, getPartitions are
implemented for the Ignite backend. The corresponding dependencies
IgniteQuery and IgniteResult are also included.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/87eddc24
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/87eddc24
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/87eddc24
Branch: refs/heads/master
Commit: 87eddc2495cdeec9b28246ea08708540c365f34e
Parents: db7540e
Author: Carlos M <ca...@gmail.com>
Authored: Thu Jul 5 23:11:38 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Thu Jul 5 23:11:38 2018 -0500
----------------------------------------------------------------------
.../apache/gora/ignite/query/IgniteQuery.java | 32 ++++++
.../apache/gora/ignite/query/IgniteResult.java | 101 +++++++++++++++++++
.../apache/gora/ignite/query/package-info.java | 20 ++++
.../apache/gora/ignite/store/IgniteStore.java | 65 +++++++++++-
.../gora/ignite/utils/IgniteSQLBuilder.java | 89 ++++++++++++++++
.../apache/gora/ignite/utils/package-info.java | 20 ++++
6 files changed, 322 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/87eddc24/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
new file mode 100644
index 0000000..b33e682
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.query;
+
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.impl.QueryBase;
+import org.apache.gora.store.DataStore;
+
+/**
+ * Ignite specific implementation of the {@link Query} interface.
+ */
+public class IgniteQuery<K, T extends PersistentBase> extends QueryBase<K, T> {
+
+ public IgniteQuery(DataStore<K, T> dataStore) {
+ super(dataStore);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/87eddc24/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
new file mode 100644
index 0000000..09f717e
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.query;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.gora.ignite.store.IgniteStore;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.apache.gora.query.Query;
+import org.apache.gora.query.impl.ResultBase;
+import org.apache.gora.store.DataStore;
+
+/**
+ * IgniteResult specific implementation of the
+ * {@link org.apache.gora.query.Result} interface.
+ */
+public class IgniteResult<K, T extends PersistentBase> extends ResultBase<K, T> {
+
+ private ResultSet resultSet;
+ private Statement st;
+ private int size;
+
+ public IgniteResult(DataStore<K, T> dataStore, Query<K, T> query) {
+ super(dataStore, query);
+ }
+
+ @Override
+ protected boolean nextInner() throws IOException {
+ try {
+ if (!resultSet.next()) {
+ return false;
+ }
+ key = ((IgniteStore<K, T>) getDataStore()).extractKey(resultSet);
+ persistent = ((IgniteStore<K, T>) getDataStore()).newInstance(resultSet, getQuery().getFields());
+ return persistent != null;
+ } catch (SQLException ex) {
+ throw new IOException(ex);
+ }
+ }
+
+ @Override
+ public float getProgress() throws IOException, InterruptedException {
+ if (resultSet == null) {
+ return 0;
+ } else if (size == 0) {
+ return 1;
+ } else {
+ return offset / (float) size;
+ }
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (resultSet != null) {
+ try {
+ resultSet.close();
+ } catch (SQLException ex) {
+ throw new IOException(ex);
+ }
+ }
+ if (st != null) {
+ try {
+ st.close();
+ } catch (SQLException ex) {
+ throw new IOException(ex);
+ }
+ }
+ }
+
+ public void setResultSet(ResultSet resultSet) throws SQLException {
+ this.resultSet = resultSet;
+ if (resultSet.last()) {
+ size = resultSet.getRow();
+ } else {
+ size = 0;
+ }
+ resultSet.beforeFirst();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/87eddc24/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
new file mode 100644
index 0000000..5e463ff
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite datastore related all classes.
+ */
+package org.apache.gora.ignite.query;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/87eddc24/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
index 497e161..d9f3527 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -28,19 +28,24 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
+import javax.sql.rowset.CachedRowSet;
+import javax.sql.rowset.RowSetFactory;
+import javax.sql.rowset.RowSetProvider;
import org.apache.avro.Schema;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.Utf8;
+import org.apache.gora.ignite.query.IgniteQuery;
+import org.apache.gora.ignite.query.IgniteResult;
import org.apache.gora.ignite.utils.IgniteSQLBuilder;
import org.apache.gora.persistency.Persistent;
import org.apache.gora.persistency.impl.PersistentBase;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
+import org.apache.gora.query.impl.PartitionQueryImpl;
import org.apache.gora.store.impl.DataStoreBase;
import org.apache.gora.util.AvroUtils;
import org.apache.gora.util.GoraException;
@@ -334,22 +339,72 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public long deleteByQuery(Query<K, T> query) throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+
+ String deleteQuery;
+ if (query.getFields() != null && query.getFields().length < igniteMapping.getFields().size()) {
+ List<String> dbFields = new ArrayList<>();
+ for (String af : query.getFields()) {
+ dbFields.add(igniteMapping.getFields().get(af).getName());
+ }
+ deleteQuery = IgniteSQLBuilder.deleteQueryFields(igniteMapping, dbFields);
+ } else {
+ deleteQuery = IgniteSQLBuilder.deleteQuery(igniteMapping);
+ }
+ String selectQueryWhere = IgniteSQLBuilder.selectQueryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
+ try (PreparedStatement stmt = connection.prepareStatement(deleteQuery + selectQueryWhere)) {
+ IgniteSQLBuilder.fillSelectQuery(stmt, query.getStartKey(), query.getEndKey());
+ stmt.executeUpdate();
+ return 0;
+ } catch (SQLException ex) {
+ throw new GoraException(ex);
+ }
}
@Override
public Result<K, T> execute(Query<K, T> query) throws GoraException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ String[] fields = getFieldsToQuery(query.getFields());
+ //Avro fields to Ignite fields
+ List<String> dbFields = new ArrayList<>();
+ for (String af : fields) {
+ dbFields.add(igniteMapping.getFields().get(af).getName());
+ }
+ String selectQuery = IgniteSQLBuilder.selectQuery(igniteMapping, dbFields);
+ String selectQueryWhere = IgniteSQLBuilder.selectQueryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
+ try {
+ PreparedStatement stmt = connection.prepareStatement(selectQuery + selectQueryWhere);
+ RowSetFactory factory = RowSetProvider.newFactory();
+ CachedRowSet rowset = factory.createCachedRowSet();
+ IgniteSQLBuilder.fillSelectQuery(stmt, query.getStartKey(), query.getEndKey());
+ ResultSet executeQuery = stmt.executeQuery();
+ rowset.populate(executeQuery);
+ IgniteResult<K, T> igniteResult = new IgniteResult<>(this, query);
+ igniteResult.setResultSet(rowset);
+ return igniteResult;
+ } catch (SQLException ex) {
+ throw new GoraException(ex);
+ }
+ }
+
+ public K extractKey(ResultSet r) throws SQLException {
+ assert igniteMapping.getPrimaryKey().size() == 1;
+ return (K) r.getObject(igniteMapping.getPrimaryKey().get(0).getName());
}
@Override
public Query<K, T> newQuery() {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ IgniteQuery<K, T> query = new IgniteQuery<>(this);
+ query.setFields(getFieldsToQuery(null));
+ return query;
}
@Override
public List<PartitionQuery<K, T>> getPartitions(Query<K, T> query) throws IOException {
- throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
+ List<PartitionQuery<K, T>> partitions = new ArrayList<>();
+ PartitionQueryImpl<K, T> partitionQuery = new PartitionQueryImpl<>(
+ query);
+ partitionQuery.setConf(getConf());
+ partitions.add(partitionQuery);
+ return partitions;
}
@Override
http://git-wip-us.apache.org/repos/asf/gora/blob/87eddc24/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
index 92014c1..4c1e314 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
@@ -146,4 +146,93 @@ public class IgniteSQLBuilder {
st.setObject(j, data[i]);
}
}
+
+ public static String selectQuery(IgniteMapping mapping, List<String> ifields) {
+ List<String> fields = new ArrayList<>();
+ for (Column c : mapping.getPrimaryKey()) {
+ fields.add(c.getName());
+ }
+ fields.addAll(ifields);
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("SELECT ");
+ for (int i = 0; i < fields.size(); i++) {
+ sqlBuilder.append(fields.get(i));
+ sqlBuilder.append(i == fields.size() - 1 ? "" : " , ");
+ }
+ sqlBuilder.append(" FROM ");
+ sqlBuilder.append(mapping.getTableName());
+ return sqlBuilder.toString();
+ }
+
+ public static String deleteQuery(IgniteMapping mapping) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("DELETE FROM ");
+ sqlBuilder.append(mapping.getTableName());
+ return sqlBuilder.toString();
+ }
+
+ public static String deleteQueryFields(IgniteMapping mapping, List<String> lsFields) {
+ StringBuilder sqlBuilder = new StringBuilder();
+ sqlBuilder.append("UPDATE ");
+ sqlBuilder.append(mapping.getTableName());
+ if (!lsFields.isEmpty()) {
+ sqlBuilder.append(" SET ");
+ }
+ for (int i = 0; i < lsFields.size(); i++) {
+ sqlBuilder.append(lsFields.get(i));
+ sqlBuilder.append(" = null");
+ sqlBuilder.append(i == lsFields.size() - 1 ? "" : " , ");
+ }
+ return sqlBuilder.toString();
+ }
+
+ public static String selectQueryWhere(IgniteMapping mapping, Object sk, Object ek, long limit) {
+ //composite keys pending
+ assert mapping.getPrimaryKey().size() == 1;
+ String keycolumn = mapping.getPrimaryKey().get(0).getName();
+ StringBuilder sqlBuilder = new StringBuilder();
+ if (sk != null || ek != null) {
+ sqlBuilder.append(" WHERE ");
+ if (sk != null && ek != null && sk.equals(ek)) {
+ sqlBuilder.append(keycolumn);
+ sqlBuilder.append("= ?");
+ } else {
+ if (sk != null) {
+ sqlBuilder.append(keycolumn);
+ sqlBuilder.append(">= ?");
+ }
+ if (sk != null && ek != null) {
+ sqlBuilder.append(" AND ");
+ }
+ if (ek != null) {
+ sqlBuilder.append(keycolumn);
+ sqlBuilder.append("<= ?");
+ }
+ }
+ }
+ if (limit > 0) {
+ sqlBuilder.append(" LIMIT " + limit);
+ }
+ return sqlBuilder.toString();
+ }
+
+ public static void fillSelectQuery(PreparedStatement st, Object sk, Object ek) throws SQLException {
+ if (sk != null || ek != null) {
+ if (sk != null && ek != null && sk.equals(ek)) {
+ st.setObject(1, sk);
+ } else {
+ if (sk != null && ek != null) {
+ st.setObject(1, sk);
+ st.setObject(2, ek);
+ } else {
+ if (sk != null) {
+ st.setObject(1, sk);
+ } else {
+ st.setObject(1, ek);
+ }
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/gora/blob/87eddc24/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
new file mode 100644
index 0000000..ce4cd7a
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains Ignite datastore related all classes.
+ */
+package org.apache.gora.ignite.utils;
\ No newline at end of file
[03/11] gora git commit: Merge branch 'master' of
https://github.com/apache/gora into GORA-535
Posted by dj...@apache.org.
Merge branch 'master' of https://github.com/apache/gora into GORA-535
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/7d671376
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/7d671376
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/7d671376
Branch: refs/heads/master
Commit: 7d671376a47a7251dad9df2e8176946b4ddb3420
Parents: 4346216 a839865
Author: Carlos M <ca...@gmail.com>
Authored: Sun Jun 3 17:35:36 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Sun Jun 3 17:35:36 2018 -0500
----------------------------------------------------------------------
.../gora/accumulo/store/AccumuloStore.java | 7 +++--
.../gora/cassandra/store/CassandraClient.java | 2 +-
.../gora/cassandra/store/CassandraStore.java | 4 +--
.../gora/persistency/impl/BeanFactoryImpl.java | 2 +-
.../persistency/ws/impl/BeanFactoryWSImpl.java | 8 +++---
.../dynamodb/store/DynamoDBNativeStore.java | 6 +++--
.../goraci/chef/ChefSoftwareProvisioning.java | 19 ++++++++------
.../rackspace/RackspaceOrchestration.java | 27 ++++++++++++--------
.../gora/infinispan/store/InfinispanClient.java | 2 +-
.../gora/infinispan/store/InfinispanStore.java | 2 +-
10 files changed, 47 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
[10/11] gora git commit: Add Ignite to the dependency management
section
Posted by dj...@apache.org.
Add Ignite to the dependency management section
The dependencies of the Ignite data store were added to the dependency
management section. Also a unused dependency (Jline) was removed from
the Ignite backend.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/8220a159
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/8220a159
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/8220a159
Branch: refs/heads/master
Commit: 8220a15915a86faa68419fe47fd78d4c349f8353
Parents: 740ab81
Author: Carlos M <ca...@gmail.com>
Authored: Sun Jul 29 23:42:29 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Sun Jul 29 23:42:29 2018 -0500
----------------------------------------------------------------------
gora-ignite/pom.xml | 8 --------
pom.xml | 23 ++++++++++++++++++++++-
2 files changed, 22 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/8220a159/gora-ignite/pom.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/pom.xml b/gora-ignite/pom.xml
index 430b8ed..8f75553 100644
--- a/gora-ignite/pom.xml
+++ b/gora-ignite/pom.xml
@@ -119,14 +119,6 @@
</dependency>
<dependency>
- <!-- Ignite mini was trying to load the Shell, which failed with the version of jline in the parent pom -->
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- <version>2.11</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/gora/blob/8220a159/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b4838f0..de30ec6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -956,6 +956,21 @@
<version>${project.version}</version>
<type>test-jar</type>
</dependency>
+
+ <!--Ignite DataStore dependencies -->
+
+ <dependency>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora-ignite</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.gora</groupId>
+ <artifactId>gora-ignite</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+
<!-- End of Internal Depednecies -->
<dependency>
@@ -1573,7 +1588,13 @@
<artifactId>testcontainers</artifactId>
<version>${test.container.version}</version>
</dependency>
-
+
+ <!-- Testing Dependencies for Ignite-->
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-indexing</artifactId>
+ <version>${ignite.version}</version>
+ </dependency>
</dependencies>
</dependencyManagement>
[05/11] gora git commit: Add mapping reading and initialization for
Ignite backend
Posted by dj...@apache.org.
Add mapping reading and initialization for Ignite backend
Basic configuration and connection classes were implemented for the
Ignite backend. The basic skeleton of the new DataStore was initialized
using mapping files and some trivial methods were implemented
(#close(), #flush(), #getSchemaName(), ...). Also, an embembed Ignite
instance was included within the test packages in order to be used in
testing.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/7545170d
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/7545170d
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/7545170d
Branch: refs/heads/master
Commit: 7545170df8047ea6127660c1750e1d0b59044e49
Parents: 7d67137
Author: Carlos M <ca...@gmail.com>
Authored: Wed Jun 6 00:03:10 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Wed Jun 6 00:08:48 2018 -0500
----------------------------------------------------------------------
gora-ignite/pom.xml | 21 +-
.../gora/ignite/encoders/BinaryEncoder.java | 191 ----
.../apache/gora/ignite/encoders/Encoder.java | 72 --
.../apache/gora/ignite/encoders/HexEncoder.java | 204 ----
.../ignite/encoders/SignedBinaryEncoder.java | 110 --
.../org/apache/gora/ignite/encoders/Utils.java | 91 --
.../gora/ignite/encoders/package-info.java | 20 -
.../apache/gora/ignite/query/IgniteQuery.java | 45 -
.../apache/gora/ignite/query/IgniteResult.java | 101 --
.../apache/gora/ignite/query/package-info.java | 21 -
.../org/apache/gora/ignite/store/Column.java | 70 ++
.../apache/gora/ignite/store/IgniteMapping.java | 44 +-
.../gora/ignite/store/IgniteMappingBuilder.java | 107 ++
.../gora/ignite/store/IgniteParameters.java | 139 +++
.../apache/gora/ignite/store/IgniteStore.java | 1013 ++----------------
.../ignite/util/FixedByteArrayOutputStream.java | 45 -
.../apache/gora/ignite/util/package-info.java | 20 -
.../gora/ignite/GoraIgniteTestDriver.java | 49 +-
.../org/apache/gora/ignite/package-info.java | 21 -
.../ignite/store/AuthenticationTokenTest.java | 90 --
.../gora/ignite/store/IgniteStoreTest.java | 87 --
.../apache/gora/ignite/store/PartitionTest.java | 96 --
.../gora/ignite/store/TestIgniteStore.java | 32 +
.../apache/gora/ignite/store/package-info.java | 21 -
.../apache/gora/ignite/util/HexEncoderTest.java | 56 -
.../ignite/util/SignedBinaryEncoderTest.java | 167 ---
.../apache/gora/ignite/util/package-info.java | 20 -
.../test/resources/gora-accumulo-mapping.xml | 59 -
.../src/test/resources/gora-ignite-mapping.xml | 42 +
gora-ignite/src/test/resources/gora.properties | 11 +-
nbactions.xml | 13 +
31 files changed, 550 insertions(+), 2528 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/pom.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/pom.xml b/gora-ignite/pom.xml
index ddd3a4a..fa31195 100644
--- a/gora-ignite/pom.xml
+++ b/gora-ignite/pom.xml
@@ -50,7 +50,7 @@
</ciManagement>
<properties>
- <ignite.version>1.7.1</ignite.version>
+ <ignite.version>2.4.0</ignite.version>
<osgi.import>*</osgi.import>
<osgi.export>org.apache.gora.ignite*;version="${project.version}";-noimport:=true</osgi.export>
</properties>
@@ -115,13 +115,6 @@
</dependency>
<dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-minicluster</artifactId>
- <version>${ignite.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
@@ -144,6 +137,12 @@
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.jdom</groupId>
+ <artifactId>jdom</artifactId>
+ <scope>compile</scope>
+ </dependency>
<!-- Logging Dependencies -->
<dependency>
@@ -171,6 +170,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-indexing</artifactId>
+ <version>2.4.0</version>
+ </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java
deleted file mode 100644
index ce6eded..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/BinaryEncoder.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.encoders;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.gora.ignite.util.FixedByteArrayOutputStream;
-
-/**
- *
- */
-public class BinaryEncoder implements Encoder {
-
- @Override
- public byte[] encodeShort(short s) throws IOException {
- return encodeShort(s, new byte[2]);
- }
-
- @Override
- public byte[] encodeShort(short s, byte[] ret) throws IOException {
- try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
- dos.writeShort(s);
- dos.close();
- return ret;
- }
- }
-
- @Override
- public short decodeShort(byte[] a) throws IOException {
- try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
- short s = dis.readShort();
- dis.close();
- return s;
- }
- }
-
- @Override
- public byte[] encodeInt(int i) throws IOException {
- return encodeInt(i, new byte[4]);
- }
-
- @Override
- public byte[] encodeInt(int i, byte[] ret) throws IOException {
- try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
- dos.writeInt(i);
- dos.close();
- return ret;
- }
- }
-
- @Override
- public int decodeInt(byte[] a) throws IOException {
- try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
- int i = dis.readInt();
- dis.close();
- return i;
- }
- }
-
- @Override
- public byte[] encodeLong(long l) throws IOException {
- return encodeLong(l, new byte[8]);
- }
-
- @Override
- public byte[] encodeLong(long l, byte[] ret) throws IOException {
- try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
- dos.writeLong(l);
- dos.close();
- return ret;
- }
- }
-
- @Override
- public long decodeLong(byte[] a) throws IOException {
- try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
- long l = dis.readLong();
- dis.close();
- return l;
- }
- }
-
- @Override
- public byte[] encodeDouble(double d) throws IOException {
- return encodeDouble(d, new byte[8]);
- }
-
- @Override
- public byte[] encodeDouble(double d, byte[] ret) throws IOException {
- try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
- long l = Double.doubleToRawLongBits(d);
- dos.writeLong(l);
- dos.close();
- return ret;
- }
- }
-
- @Override
- public double decodeDouble(byte[] a) throws IOException {
- try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
- long l = dis.readLong();
- dis.close();
- return Double.longBitsToDouble(l);
- }
- }
-
- @Override
- public byte[] encodeFloat(float d) throws IOException {
- return encodeFloat(d, new byte[4]);
- }
-
- @Override
- public byte[] encodeFloat(float f, byte[] ret) throws IOException {
- try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
- int i = Float.floatToRawIntBits(f);
- dos.writeInt(i);
- return ret;
- }
- }
-
- @Override
- public float decodeFloat(byte[] a) throws IOException {
- try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
- int i = dis.readInt();
- return Float.intBitsToFloat(i);
- }
- }
-
- @Override
- public byte[] encodeByte(byte b, byte[] ret) {
- ret[0] = 0;
- return ret;
- }
-
- @Override
- public byte[] encodeByte(byte b) {
- return encodeByte(b, new byte[1]);
- }
-
- @Override
- public byte decodeByte(byte[] a) {
- return a[0];
- }
-
- @Override
- public boolean decodeBoolean(byte[] a) throws IOException {
- try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(a))){
- return dis.readBoolean();
- }
- }
-
- @Override
- public byte[] encodeBoolean(boolean b) throws IOException {
- return encodeBoolean(b, new byte[1]);
- }
-
- @Override
- public byte[] encodeBoolean(boolean b, byte[] ret) throws IOException {
- try (DataOutputStream dos = new DataOutputStream(new FixedByteArrayOutputStream(ret))){
- dos.writeBoolean(b);
- return ret;
- }
- }
-
- @Override
- public byte[] lastPossibleKey(int size, byte[] er) {
- return Utils.lastPossibleKey(size, er);
- }
-
- @Override
- public byte[] followingKey(int size, byte[] per) {
- return Utils.followingKey(size, per);
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java
deleted file mode 100644
index deea4a7..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Encoder.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.encoders;
-
-import java.io.IOException;
-
-/**
- *
- */
-public interface Encoder {
-
- public byte[] encodeByte(byte b, byte[] ret);
-
- public byte[] encodeByte(byte b);
-
- public byte decodeByte(byte[] a);
-
- public byte[] encodeShort(short s) throws IOException;
-
- public byte[] encodeShort(short s, byte[] ret) throws IOException;
-
- public short decodeShort(byte[] a) throws IOException;
-
- public byte[] encodeInt(int i) throws IOException;
-
- public byte[] encodeInt(int i, byte[] ret) throws IOException;
-
- public int decodeInt(byte[] a) throws IOException;
-
- public byte[] encodeLong(long l) throws IOException;
-
- public byte[] encodeLong(long l, byte[] ret) throws IOException;
-
- public long decodeLong(byte[] a) throws IOException;
-
- public byte[] encodeDouble(double d) throws IOException;
-
- public byte[] encodeDouble(double d, byte[] ret) throws IOException;
-
- public double decodeDouble(byte[] a) throws IOException;
-
- public byte[] encodeFloat(float d) throws IOException;
-
- public byte[] encodeFloat(float f, byte[] ret) throws IOException;
-
- public float decodeFloat(byte[] a) throws IOException;
-
- public boolean decodeBoolean(byte[] val) throws IOException;
-
- public byte[] encodeBoolean(boolean b) throws IOException;
-
- public byte[] encodeBoolean(boolean b, byte[] ret) throws IOException;
-
- byte[] followingKey(int size, byte[] per);
-
- byte[] lastPossibleKey(int size, byte[] er);
-
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java
deleted file mode 100644
index 8568ba9..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/HexEncoder.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.encoders;
-
-/**
- * Encodes data in a ascii hex representation
- */
-
-public class HexEncoder implements Encoder {
-
- private byte[] chars = new byte[] {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
-
- private void encode(byte[] a, long l) {
- for (int i = a.length - 1; i >= 0; i--) {
- a[i] = chars[(int) (l & 0x0f)];
- l = l >>> 4;
- }
- }
-
- private int fromChar(byte b) {
- if (b >= '0' && b <= '9') {
- return b - '0';
- } else if (b >= 'a' && b <= 'f') {
- return b - 'a' + 10;
- }
-
- throw new IllegalArgumentException("Bad char " + b);
- }
-
- private long decode(byte[] a) {
- long b = 0;
- for (byte anA : a) {
- b = b << 4;
- b |= fromChar(anA);
- }
-
- return b;
- }
-
- @Override
- public byte[] encodeByte(byte b, byte[] ret) {
- encode(ret, 0xff & b);
- return ret;
- }
-
- @Override
- public byte[] encodeByte(byte b) {
- return encodeByte(b, new byte[2]);
- }
-
- @Override
- public byte decodeByte(byte[] a) {
- return (byte) decode(a);
- }
-
- @Override
- public byte[] encodeShort(short s) {
- return encodeShort(s, new byte[4]);
- }
-
- @Override
- public byte[] encodeShort(short s, byte[] ret) {
- encode(ret, 0xffff & s);
- return ret;
- }
-
- @Override
- public short decodeShort(byte[] a) {
- return (short) decode(a);
- }
-
- @Override
- public byte[] encodeInt(int i) {
- return encodeInt(i, new byte[8]);
- }
-
- @Override
- public byte[] encodeInt(int i, byte[] ret) {
- encode(ret, i);
- return ret;
- }
-
- @Override
- public int decodeInt(byte[] a) {
- return (int) decode(a);
- }
-
- @Override
- public byte[] encodeLong(long l) {
- return encodeLong(l, new byte[16]);
- }
-
- @Override
- public byte[] encodeLong(long l, byte[] ret) {
- encode(ret, l);
- return ret;
- }
-
- @Override
- public long decodeLong(byte[] a) {
- return decode(a);
- }
-
- @Override
- public byte[] encodeDouble(double d) {
- return encodeDouble(d, new byte[16]);
- }
-
- @Override
- public byte[] encodeDouble(double d, byte[] ret) {
- return encodeLong(Double.doubleToRawLongBits(d), ret);
- }
-
- @Override
- public double decodeDouble(byte[] a) {
- return Double.longBitsToDouble(decodeLong(a));
- }
-
- @Override
- public byte[] encodeFloat(float d) {
- return encodeFloat(d, new byte[16]);
- }
-
- @Override
- public byte[] encodeFloat(float d, byte[] ret) {
- return encodeInt(Float.floatToRawIntBits(d), ret);
- }
-
- @Override
- public float decodeFloat(byte[] a) {
- return Float.intBitsToFloat(decodeInt(a));
- }
-
- @Override
- public boolean decodeBoolean(byte[] val) {
- return decodeByte(val) == 1;
- }
-
- @Override
- public byte[] encodeBoolean(boolean b) {
- return encodeBoolean(b, new byte[2]);
- }
-
- @Override
- public byte[] encodeBoolean(boolean b, byte[] ret) {
- if (b)
- encode(ret, 1);
- else
- encode(ret, 0);
-
- return ret;
- }
-
- private byte[] toBinary(byte[] hex) {
- byte[] bin = new byte[(hex.length / 2) + (hex.length % 2)];
-
- int j = 0;
- for (int i = 0; i < bin.length; i++) {
- bin[i] = (byte) (fromChar(hex[j++]) << 4);
- if (j >= hex.length)
- break;
- bin[i] |= (byte) fromChar(hex[j++]);
- }
-
- return bin;
- }
-
- private byte[] fromBinary(byte[] bin) {
- byte[] hex = new byte[bin.length * 2];
-
- int j = 0;
- for (byte aBin : bin) {
- hex[j++] = chars[0x0f & (aBin >>> 4)];
- hex[j++] = chars[0x0f & aBin];
- }
-
- return hex;
- }
-
- @Override
- public byte[] followingKey(int size, byte[] per) {
- return fromBinary(Utils.followingKey(size, toBinary(per)));
- }
-
- @Override
- public byte[] lastPossibleKey(int size, byte[] er) {
- return fromBinary(Utils.lastPossibleKey(size, toBinary(er)));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java
deleted file mode 100644
index a8216f4..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/SignedBinaryEncoder.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.encoders;
-
-import java.io.IOException;
-
-/**
- * This class transforms this bits within a primitive type so that
- * the bit representation sorts correctly lexographicaly. Primarily
- * it does some simple transformations so that negative numbers sort
- * before positive numbers, when compared lexographically.
- */
-public class SignedBinaryEncoder extends BinaryEncoder {
-
- @Override
- public byte[] encodeShort(short s, byte[] ret) throws IOException{
- s = (short)((s & 0xffff) ^ 0x8000);
- return super.encodeShort(s, ret);
- }
-
- @Override
- public short decodeShort(byte[] a) throws IOException{
- short s = super.decodeShort(a);
- s = (short)((s & 0xffff) ^ 0x8000);
- return s;
- }
-
- @Override
- public byte[] encodeInt(int i, byte[] ret) throws IOException{
- i = i ^ 0x80000000;
- return super.encodeInt(i, ret);
- }
-
- @Override
- public int decodeInt(byte[] a) throws IOException{
- int i = super.decodeInt(a);
- i = i ^ 0x80000000;
- return i;
- }
-
- @Override
- public byte[] encodeLong(long l, byte[] ret) throws IOException{
- l = l ^ 0x8000000000000000L;
- return super.encodeLong(l, ret);
- }
-
- @Override
- public long decodeLong(byte[] a) throws IOException {
- long l = super.decodeLong(a);
- l = l ^ 0x8000000000000000L;
- return l;
- }
-
- @Override
- public byte[] encodeDouble(double d, byte[] ret) throws IOException {
- long l = Double.doubleToRawLongBits(d);
- if(l < 0)
- l = ~l;
- else
- l = l ^ 0x8000000000000000L;
- return super.encodeLong(l,ret);
- }
-
- @Override
- public double decodeDouble(byte[] a) throws IOException{
- long l = super.decodeLong(a);
- if(l < 0)
- l = l ^ 0x8000000000000000L;
- else
- l = ~l;
- return Double.longBitsToDouble(l);
- }
-
- @Override
- public byte[] encodeFloat(float f, byte[] ret) throws IOException {
- int i = Float.floatToRawIntBits(f);
- if(i < 0)
- i = ~i;
- else
- i = i ^ 0x80000000;
-
- return super.encodeInt(i, ret);
-
- }
-
- @Override
- public float decodeFloat(byte[] a) throws IOException{
- int i = super.decodeInt(a);
- if(i < 0)
- i = i ^ 0x80000000;
- else
- i = ~i;
- return Float.intBitsToFloat(i);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java
deleted file mode 100644
index 8a5980c..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/Utils.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.encoders;
-
-import java.math.BigInteger;
-import java.util.Arrays;
-
-/**
- *
- */
-public class Utils {
- private static BigInteger newPositiveBigInteger(byte[] er) {
- byte[] copy = new byte[er.length + 1];
- System.arraycopy(er, 0, copy, 1, er.length);
- BigInteger bi = new BigInteger(copy);
- return bi;
- }
-
- public static byte[] lastPossibleKey(int size, byte[] er) {
- if (size == er.length)
- return er;
-
- if (er.length > size)
- throw new IllegalArgumentException();
-
- BigInteger bi = newPositiveBigInteger(er);
- if (bi.equals(BigInteger.ZERO))
- throw new IllegalArgumentException("Nothing comes before zero");
-
- bi = bi.subtract(BigInteger.ONE);
-
- byte[] ret = new byte[size];
- Arrays.fill(ret, (byte) 0xff);
-
- System.arraycopy(getBytes(bi, er.length), 0, ret, 0, er.length);
-
- return ret;
- }
-
- private static byte[] getBytes(BigInteger bi, int minLen) {
- byte[] ret = bi.toByteArray();
-
- if (ret[0] == 0) {
- // remove leading 0 that makes num positive
- byte[] copy = new byte[ret.length - 1];
- System.arraycopy(ret, 1, copy, 0, copy.length);
- ret = copy;
- }
-
- // leading digits are dropped
- byte[] copy = new byte[minLen];
- if (bi.compareTo(BigInteger.ZERO) < 0) {
- Arrays.fill(copy, (byte) 0xff);
- }
- System.arraycopy(ret, 0, copy, minLen - ret.length, ret.length);
-
- return copy;
- }
-
- public static byte[] followingKey(int size, byte[] per) {
-
- if (per.length > size)
- throw new IllegalArgumentException();
-
- if (size == per.length) {
- // add one
- BigInteger bi = new BigInteger(per);
- bi = bi.add(BigInteger.ONE);
- if (bi.equals(BigInteger.ZERO)) {
- throw new IllegalArgumentException("Wrapped");
- }
- return getBytes(bi, size);
- } else {
- return Arrays.copyOf(per, size);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java
deleted file mode 100644
index 574aa24..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/encoders/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This package contains Ignite store related util classes for encoder.
- */
-package org.apache.gora.ignite.encoders;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
deleted file mode 100644
index 85a59c9..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.query;
-
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.query.impl.QueryBase;
-import org.apache.gora.store.DataStore;
-
-/**
- * Ignite specific implementation of the {@link org.apache.gora.query.Query} interface.
- */
-public class IgniteQuery<K,T extends PersistentBase> extends QueryBase<K,T> {
-
- /**
- * Constructor for the query
- */
- public IgniteQuery() {
- super(null);
- }
-
- /**
- * Constructor for the query
- *
- * @param dataStore Data store used
- *
- */
- public IgniteQuery(DataStore<K,T> dataStore) {
- super(dataStore);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
deleted file mode 100644
index 416e650..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gora.ignite.query;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.Map.Entry;
-
-import org.apache.ignite.core.client.RowIterator;
-import org.apache.ignite.core.client.Scanner;
-import org.apache.ignite.core.data.ByteSequence;
-import org.apache.ignite.core.data.Key;
-import org.apache.ignite.core.data.Value;
-import org.apache.gora.ignite.store.IgniteStore;
-import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.query.Query;
-import org.apache.gora.query.impl.ResultBase;
-import org.apache.gora.store.DataStore;
-
-/**
- * Ignite specific implementation of the {@link org.apache.gora.query.Result} interface.
- */
-public class IgniteResult<K,T extends PersistentBase> extends ResultBase<K,T> {
-
- private RowIterator iterator;
-
- /**
- * Gets the data store used
- */
- public IgniteStore<K,T> getDataStore() {
- return (IgniteStore<K,T>) super.getDataStore();
- }
-
- /**
- * @param dataStore
- * @param query
- * @param scanner
- */
- public IgniteResult(DataStore<K,T> dataStore, Query<K,T> query, Scanner scanner) {
- super(dataStore, query);
-
- if (this.limit > 0) {
- scanner.setBatchSize((int) this.limit);
- }
- iterator = new RowIterator(scanner.iterator());
- }
-
- /**
- * Gets the items reading progress
- */
- @Override
- public float getProgress() throws IOException {
- if (this.limit != -1) {
- return (float) this.offset / (float) this.limit;
- } else {
- return 0;
- }
- }
-
- @Override
- public void close() throws IOException {
-
- }
-
- /**
- * Gets the next item
- */
- @Override
- protected boolean nextInner() throws IOException {
-
- if (!iterator.hasNext())
- return false;
-
- key = null;
-
- Iterator<Entry<Key,Value>> nextRow = iterator.next();
- ByteSequence row = getDataStore().populate(nextRow, persistent);
- key = ((IgniteStore<K, T>) dataStore).fromBytes(getKeyClass(), row.toArray());
-
- return true;
- }
-
- @Override
- public int size() {
- return (int) this.limit;
- }
-}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
deleted file mode 100644
index b1a306c..0000000
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * This package contains all the Ignite store query representation class as well as Result set representing class
- * when query is executed over the Ignite dataStore.
- */
-package org.apache.gora.ignite.query;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
new file mode 100644
index 0000000..d06591e
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+public class Column {
+
+ private String name;
+ private FieldType dataType;
+
+ public Column(String name, FieldType dataType) {
+ this.name = name;
+ this.dataType = dataType;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public FieldType getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(FieldType dataType) {
+ this.dataType = dataType;
+ }
+
+
+
+ /**
+ * For a more detailed list of data types supported by Ignite and its
+ * equivalents in Java refer to
+ * https://apacheignite-sql.readme.io/docs/data-types
+ */
+ public static enum FieldType {
+ BOOLEAN,
+ INT,
+ TINYINT,
+ SMALLINT,
+ BIGINT,
+ DECIMAL,
+ DOUBLE,
+ REAL,
+ TIME,
+ DATE,
+ TIMESTAMP,
+ VARCHAR,
+ CHAR,
+ UUID,
+ BINARY,
+ ARRAY
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
index b46c063..2e59da1 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
@@ -17,28 +17,44 @@
package org.apache.gora.ignite.store;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import org.apache.ignite.core.util.Pair;
-import org.apache.hadoop.io.Text;
-
/**
* Mapping definitions for Ignite.
*/
public class IgniteMapping {
- /**
- * A map of field names to Field objects containing schema's fields
- */
- Map<String,Pair<Text,Text>> fieldMap = new HashMap<>();
+ private String tableName;
+ private Map<String, Column> fields;
+ private List<String> primaryKey;
+
+ public IgniteMapping() {
+ fields = new HashMap<>();
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public Map<String, Column> getFields() {
+ return fields;
+ }
+
+ public void setFields(Map<String, Column> fields) {
+ this.fields = fields;
+ }
- /**
- * Look up the column associated to the Avro field.
- */
- Map<Pair<Text,Text>,String> columnMap = new HashMap<>();
+ public List<String> getPrimaryKey() {
+ return primaryKey;
+ }
- Map<String,String> tableConfig = new HashMap<>();
- String tableName;
- String encoder;
+ public void setPrimaryKey(List<String> primaryKey) {
+ this.primaryKey = primaryKey;
+ }
}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
new file mode 100644
index 0000000..81a3fae
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import com.google.inject.ConfigurationException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.gora.persistency.impl.PersistentBase;
+import org.jdom.Document;
+import org.jdom.Element;
+import org.jdom.JDOMException;
+import org.jdom.input.SAXBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Builder for Mapping definitions of Ignite.
+ */
+public class IgniteMappingBuilder<K, T extends PersistentBase> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(IgniteMappingBuilder.class);
+ /**
+ * Mapping instance being built
+ */
+ private IgniteMapping igniteMapping;
+
+ private final IgniteStore<K, T> dataStore;
+
+ public IgniteMappingBuilder(final IgniteStore<K, T> store) {
+ this.igniteMapping = new IgniteMapping();
+ this.dataStore = store;
+ }
+
+ public IgniteMapping getIgniteMapping() {
+ return igniteMapping;
+ }
+
+ public void setIgniteMapping(IgniteMapping igniteMapping) {
+ this.igniteMapping = igniteMapping;
+ }
+
+ public void readMappingFile(String mappingFile) {
+ try {
+ SAXBuilder saxBuilder = new SAXBuilder();
+ InputStream inputStream = getClass().getClassLoader().getResourceAsStream(mappingFile);
+ if (inputStream == null) {
+ LOG.error("Mapping file '{}' could not be found!", mappingFile);
+ throw new IOException("Mapping file '" + mappingFile + "' could not be found!");
+ }
+ Document document = saxBuilder.build(inputStream);
+ if (document == null) {
+ LOG.error("Mapping file '{}' could not be found!", mappingFile);
+ throw new IOException("Mapping file '" + mappingFile + "' could not be found!");
+ }
+ List<Element> classes = document.getRootElement().getChildren("class");
+ for (Element classElement : classes) {
+ if (classElement.getAttributeValue("keyClass").equals(
+ dataStore.getKeyClass().getCanonicalName())
+ && classElement.getAttributeValue("name").equals(
+ dataStore.getPersistentClass().getCanonicalName())) {
+ final String tableNameFromMapping = classElement.getAttributeValue("table");
+ String tableName = dataStore.getSchemaName(tableNameFromMapping, dataStore.getPersistentClass());
+ igniteMapping.setTableName(tableName);
+ List<Element> prColumns = classElement.getChildren("primarykey");
+ List<String> prFields = new ArrayList<>();
+ for (Element aPrimaryKey : prColumns) {
+ prFields.add(aPrimaryKey.getAttributeValue("column"));
+ }
+ igniteMapping.setPrimaryKey(prFields);
+ List<Element> fields = classElement.getChildren("field");
+ Map<String, Column> mp = new HashMap<>();
+ for (Element field : fields) {
+ String fieldName = field.getAttributeValue("name");
+ String columnName = field.getAttributeValue("column");
+ String columnType = field.getAttributeValue("type");
+ mp.put(fieldName, new Column(columnName, Column.FieldType.valueOf(columnType)));
+ }
+ igniteMapping.setFields(mp);
+ break;
+ }
+ }
+
+ } catch (IOException | JDOMException | ConfigurationException e) {
+ throw new RuntimeException(e);
+ }
+ LOG.info("Gora Ignite mapping file was read successfully.");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/7545170d/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
new file mode 100644
index 0000000..15b5bae
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright 2018 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.store;
+
+import java.util.Properties;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Parameters definitions for Ignite.
+ */
+public class IgniteParameters {
+
+ /**
+ * Property indicating the Ignite Schema to be used
+ */
+ public static final String PROP_SCHEMA = "gora.datastore.ignite.schema";
+
+ /**
+ * Property indicating the Ignite Cluster Node IP
+ */
+ public static final String PROP_HOST = "gora.datastore.ignite.host";
+
+ /**
+ * Property indicating the port used by the Ignite Server
+ */
+ public static final String PROP_PORT = "gora.datastore.ignite.port";
+
+ /**
+ * Property indicating the username to connect to the server
+ */
+ public static final String PROP_USER = "gora.datastore.ignite.user";
+
+ /**
+ * Property indicating the password to connect to the server
+ */
+ public static final String PROP_PASSWORD = "gora.datastore.ignite.password";
+
+ /**
+ * Property indicating additional JDBC options
+ */
+ public static final String PROP_ADDITIONALS = "gora.datastore.ignite.additionalConfigurations";
+
+ private String host;
+ private String port;
+ private String schema;
+ private String user;
+ private String password;
+ private String additionalConfigurations;
+
+ /**
+ *
+ * @param host
+ * @param port Optional port for Ignite Server
+ * @param user Optional username for Ignite
+ * @param password Optional password for Ignite
+ * @param additionalConfigurations Optional additional configurations for
+ * Ignite
+ */
+ private IgniteParameters(String host, String port, String schema, String user, String password, String additionalConfigurations) {
+ this.host = host;
+ this.port = port;
+ this.schema = schema;
+ this.user = user;
+ this.password = password;
+ this.additionalConfigurations = additionalConfigurations;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ public String getPort() {
+ return port;
+ }
+
+ public void setPort(String port) {
+ this.port = port;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getAdditionalConfigurations() {
+ return additionalConfigurations;
+ }
+
+ public void setAdditionalConfigurations(String additionalConfigurations) {
+ this.additionalConfigurations = additionalConfigurations;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ public static IgniteParameters load(Properties properties, Configuration conf) {
+ return new IgniteParameters(
+ properties.getProperty(PROP_HOST, "localhost"),
+ properties.getProperty(PROP_PORT, "10800"),
+ properties.getProperty(PROP_SCHEMA, null),
+ properties.getProperty(PROP_USER, null),
+ properties.getProperty(PROP_PASSWORD, null),
+ properties.getProperty(PROP_ADDITIONALS, null));
+ }
+
+}
[09/11] gora git commit: Improve Ignite Datastore to better suit
Gora's coding standards
Posted by dj...@apache.org.
Improve Ignite Datastore to better suit Gora's coding standards
Formatting and documentation were improved in multiple classes. Also
dependencies' versions were parameterized within parent pom.xml files
and other minor improvements were implemented throughout the project.
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/740ab814
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/740ab814
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/740ab814
Branch: refs/heads/master
Commit: 740ab8145ac6c004d2b10ea46ea0f5c2a0a9f71f
Parents: 87eddc2
Author: Carlos M <ca...@gmail.com>
Authored: Thu Jul 26 23:57:32 2018 -0500
Committer: Carlos M <ca...@gmail.com>
Committed: Thu Jul 26 23:57:32 2018 -0500
----------------------------------------------------------------------
.gitignore | 1 +
gora-ignite/pom.xml | 4 +-
gora-ignite/src/examples/java/.gitignore | 15 --
.../org/apache/gora/ignite/package-info.java | 4 +-
.../apache/gora/ignite/query/IgniteQuery.java | 2 +-
.../apache/gora/ignite/query/IgniteResult.java | 10 +-
.../apache/gora/ignite/query/package-info.java | 2 +-
.../org/apache/gora/ignite/store/Column.java | 2 +-
.../apache/gora/ignite/store/IgniteMapping.java | 2 +-
.../gora/ignite/store/IgniteMappingBuilder.java | 5 +-
.../gora/ignite/store/IgniteParameters.java | 23 +--
.../apache/gora/ignite/store/IgniteStore.java | 59 +++---
.../apache/gora/ignite/store/package-info.java | 2 +-
.../ignite/utils/IgniteBackendConstants.java | 36 ++++
.../gora/ignite/utils/IgniteSQLBuilder.java | 193 +++++++++++++++----
.../apache/gora/ignite/utils/package-info.java | 2 +-
.../config/java.util.logging.properties | 74 +++++++
pom.xml | 2 +
18 files changed, 316 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 2aa54cc..d4f78bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,4 @@ ivy/ivy*.jar
**/gora-solr-5/src/test/conf/solr/collection1/data/
**/_rest_managed.json
**/gora-solr/src/test/conf/solr-managed-schema/collection1/data/
+**/nbactions.xml
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/pom.xml
----------------------------------------------------------------------
diff --git a/gora-ignite/pom.xml b/gora-ignite/pom.xml
index fa31195..430b8ed 100644
--- a/gora-ignite/pom.xml
+++ b/gora-ignite/pom.xml
@@ -50,7 +50,6 @@
</ciManagement>
<properties>
- <ignite.version>2.4.0</ignite.version>
<osgi.import>*</osgi.import>
<osgi.export>org.apache.gora.ignite*;version="${project.version}";-noimport:=true</osgi.export>
</properties>
@@ -174,9 +173,8 @@
<dependency>
<groupId>org.apache.ignite</groupId>
<artifactId>ignite-indexing</artifactId>
- <version>2.4.0</version>
+ <version>${ignite.version}</version>
</dependency>
-
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/examples/java/.gitignore
----------------------------------------------------------------------
diff --git a/gora-ignite/src/examples/java/.gitignore b/gora-ignite/src/examples/java/.gitignore
deleted file mode 100644
index 09697dc..0000000
--- a/gora-ignite/src/examples/java/.gitignore
+++ /dev/null
@@ -1,15 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
index a7fa7ab..4036748 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -17,4 +17,4 @@
/**
* This package contains Ignite datastore related all classes.
*/
-package org.apache.gora.ignite;
\ No newline at end of file
+package org.apache.gora.ignite;
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
index b33e682..7ad9c61 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteQuery.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
index 09f717e..e5c0d46 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/IgniteResult.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -33,7 +33,6 @@ import org.apache.gora.store.DataStore;
public class IgniteResult<K, T extends PersistentBase> extends ResultBase<K, T> {
private ResultSet resultSet;
- private Statement st;
private int size;
public IgniteResult(DataStore<K, T> dataStore, Query<K, T> query) {
@@ -79,13 +78,6 @@ public class IgniteResult<K, T extends PersistentBase> extends ResultBase<K, T>
throw new IOException(ex);
}
}
- if (st != null) {
- try {
- st.close();
- } catch (SQLException ex) {
- throw new IOException(ex);
- }
- }
}
public void setResultSet(ResultSet resultSet) throws SQLException {
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
index 5e463ff..fa52d9c 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/query/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
index d06591e..ee7d6d6 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/Column.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
index c0e7a98..f0189ec 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMapping.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
index 3d245ea..4d397c5 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteMappingBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -70,6 +70,7 @@ public class IgniteMappingBuilder<K, T extends PersistentBase> {
LOG.error("Mapping file '{}' could not be found!", mappingFile);
throw new IOException("Mapping file '" + mappingFile + "' could not be found!");
}
+ @SuppressWarnings("unchecked")
List<Element> classes = document.getRootElement().getChildren("class");
for (Element classElement : classes) {
if (classElement.getAttributeValue("keyClass").equals(
@@ -79,6 +80,7 @@ public class IgniteMappingBuilder<K, T extends PersistentBase> {
final String tableNameFromMapping = classElement.getAttributeValue("table");
String tableName = dataStore.getSchemaName(tableNameFromMapping, dataStore.getPersistentClass());
igniteMapping.setTableName(tableName);
+ @SuppressWarnings("unchecked")
List<Element> prColumns = classElement.getChildren("primarykey");
List<Column> prFields = new ArrayList<>();
for (Element aPrimaryKey : prColumns) {
@@ -87,6 +89,7 @@ public class IgniteMappingBuilder<K, T extends PersistentBase> {
prFields.add(new Column(name, Column.FieldType.valueOf(type)));
}
igniteMapping.setPrimaryKey(prFields);
+ @SuppressWarnings("unchecked")
List<Element> fields = classElement.getChildren("field");
Map<String, Column> mp = new HashMap<>();
for (Element field : fields) {
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
index 15b5bae..f667b40 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteParameters.java
@@ -1,11 +1,12 @@
/*
- * Copyright 2018 The Apache Software Foundation.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,6 +17,7 @@
package org.apache.gora.ignite.store;
import java.util.Properties;
+import org.apache.gora.ignite.utils.IgniteBackendConstants;
import org.apache.hadoop.conf.Configuration;
/**
@@ -61,8 +63,7 @@ public class IgniteParameters {
private String additionalConfigurations;
/**
- *
- * @param host
+ * @param host Hostname/IP of the Ignite Server
* @param port Optional port for Ignite Server
* @param user Optional username for Ignite
* @param password Optional password for Ignite
@@ -126,10 +127,10 @@ public class IgniteParameters {
this.schema = schema;
}
- public static IgniteParameters load(Properties properties, Configuration conf) {
+ public static IgniteParameters load(Properties properties) {
return new IgniteParameters(
- properties.getProperty(PROP_HOST, "localhost"),
- properties.getProperty(PROP_PORT, "10800"),
+ properties.getProperty(PROP_HOST, IgniteBackendConstants.DEFAULT_IGNITE_HOST),
+ properties.getProperty(PROP_PORT, IgniteBackendConstants.DEFAULT_IGNITE_PORT),
properties.getProperty(PROP_SCHEMA, null),
properties.getProperty(PROP_USER, null),
properties.getProperty(PROP_PASSWORD, null),
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
index d9f3527..a17b3a0 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/IgniteStore.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -39,6 +39,7 @@ import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.avro.util.Utf8;
import org.apache.gora.ignite.query.IgniteQuery;
import org.apache.gora.ignite.query.IgniteResult;
+import org.apache.gora.ignite.utils.IgniteBackendConstants;
import org.apache.gora.ignite.utils.IgniteSQLBuilder;
import org.apache.gora.persistency.Persistent;
import org.apache.gora.persistency.impl.PersistentBase;
@@ -61,33 +62,24 @@ import org.slf4j.LoggerFactory;
*/
public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T> {
- public static final Logger LOG = LoggerFactory.getLogger(IgniteStore.class);
+ private static final Logger LOG = LoggerFactory.getLogger(IgniteStore.class);
private static final String PARSE_MAPPING_FILE_KEY = "gora.ignite.mapping.file";
private static final String DEFAULT_MAPPING_FILE = "gora-ignite-mapping.xml";
private IgniteParameters igniteParameters;
private IgniteMapping igniteMapping;
private Connection connection;
-
- /*
- * Create a threadlocal map for the datum readers and writers, because they
- * are not thread safe, at least not before Avro 1.4.0 (See AVRO-650). When
- * they are thread safe, it is possible to maintain a single reader and writer
- * pair for every schema, instead of one for every thread.
- */
- public static final ConcurrentHashMap<Schema, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<>();
-
- public static final ConcurrentHashMap<Schema, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<>();
+ private static final ConcurrentHashMap<Schema, SpecificDatumReader<?>> readerMap = new ConcurrentHashMap<>();
+ private static final ConcurrentHashMap<Schema, SpecificDatumWriter<?>> writerMap = new ConcurrentHashMap<>();
@Override
public void initialize(Class<K> keyClass, Class<T> persistentClass, Properties properties) throws GoraException {
-
try {
super.initialize(keyClass, persistentClass, properties);
- IgniteMappingBuilder builder = new IgniteMappingBuilder(this);
+ IgniteMappingBuilder<K, T> builder = new IgniteMappingBuilder<K, T>(this);
builder.readMappingFile(getConf().get(PARSE_MAPPING_FILE_KEY, DEFAULT_MAPPING_FILE));
igniteMapping = builder.getIgniteMapping();
- igniteParameters = IgniteParameters.load(properties, conf);
- connection = acquiereConnection();
+ igniteParameters = IgniteParameters.load(properties);
+ connection = acquireConnection();
LOG.info("Ignite store was successfully initialized");
} catch (ClassNotFoundException | SQLException ex) {
LOG.error("Error while initializing Ignite store", ex);
@@ -95,7 +87,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
}
}
- private Connection acquiereConnection() throws ClassNotFoundException, SQLException {
+ private Connection acquireConnection() throws ClassNotFoundException, SQLException {
Class.forName("org.apache.ignite.IgniteJdbcThinDriver");
StringBuilder urlBuilder = new StringBuilder();
urlBuilder.append("jdbc:ignite:thin://");
@@ -115,8 +107,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
if (igniteParameters.getAdditionalConfigurations() != null) {
urlBuilder.append(igniteParameters.getAdditionalConfigurations());
}
- Connection conn = DriverManager.getConnection(urlBuilder.toString());
- return conn;
+ return DriverManager.getConnection(urlBuilder.toString());
}
@Override
@@ -167,25 +158,19 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
@Override
public boolean schemaExists() throws GoraException {
- boolean exists = false;
try (Statement stmt = connection.createStatement()) {
String tableExistsSQL = IgniteSQLBuilder.tableExists(igniteMapping.getTableName());
ResultSet executeQuery = stmt.executeQuery(tableExistsSQL);
executeQuery.close();
- exists = true;
+ return true;
} catch (SQLException ex) {
- /**
- * a 42000 error code is thrown by Ignite when a non-existent table
- * queried. More details:
- * https://apacheignite-sql.readme.io/docs/jdbc-error-codes
- */
- if (ex.getSQLState() != null && ex.getSQLState().equals("42000")) {
- exists = false;
+ if (ex.getSQLState() != null
+ && ex.getSQLState().equals(IgniteBackendConstants.DEFAULT_IGNITE_TABLE_NOT_EXISTS_CODE)) {
+ return false;
} else {
throw new GoraException(ex);
}
}
- return exists;
}
@Override
@@ -240,6 +225,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
return persistent;
}
+ @SuppressWarnings("unchecked")
private Object deserializeFieldValue(Schema.Field field, Schema fieldSchema,
Object igniteValue, T persistent) throws IOException {
Object fieldValue = null;
@@ -298,7 +284,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
Object v = obj.get(field.pos());
if (get != null && v != null) {
Schema fieldSchema = field.schema();
- Object serializedObj = serializeFieldValue(get, fieldSchema, v);
+ Object serializedObj = serializeFieldValue(fieldSchema, v);
data.put(get, serializedObj);
}
}
@@ -350,7 +336,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
} else {
deleteQuery = IgniteSQLBuilder.deleteQuery(igniteMapping);
}
- String selectQueryWhere = IgniteSQLBuilder.selectQueryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
+ String selectQueryWhere = IgniteSQLBuilder.queryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
try (PreparedStatement stmt = connection.prepareStatement(deleteQuery + selectQueryWhere)) {
IgniteSQLBuilder.fillSelectQuery(stmt, query.getStartKey(), query.getEndKey());
stmt.executeUpdate();
@@ -369,9 +355,8 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
dbFields.add(igniteMapping.getFields().get(af).getName());
}
String selectQuery = IgniteSQLBuilder.selectQuery(igniteMapping, dbFields);
- String selectQueryWhere = IgniteSQLBuilder.selectQueryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
- try {
- PreparedStatement stmt = connection.prepareStatement(selectQuery + selectQueryWhere);
+ String selectQueryWhere = IgniteSQLBuilder.queryWhere(igniteMapping, query.getStartKey(), query.getEndKey(), query.getLimit());
+ try (PreparedStatement stmt = connection.prepareStatement(selectQuery + selectQueryWhere)) {
RowSetFactory factory = RowSetProvider.newFactory();
CachedRowSet rowset = factory.createCachedRowSet();
IgniteSQLBuilder.fillSelectQuery(stmt, query.getStartKey(), query.getEndKey());
@@ -385,6 +370,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
}
}
+ @SuppressWarnings("unchecked")
public K extractKey(ResultSet r) throws SQLException {
assert igniteMapping.getPrimaryKey().size() == 1;
return (K) r.getObject(igniteMapping.getPrimaryKey().get(0).getName());
@@ -422,7 +408,8 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
}
}
- private Object serializeFieldValue(Column get, Schema fieldSchema, Object fieldValue) {
+ @SuppressWarnings("unchecked")
+ private Object serializeFieldValue(Schema fieldSchema, Object fieldValue) {
Object output = fieldValue;
switch (fieldSchema.getType()) {
case ARRAY:
@@ -442,7 +429,7 @@ public class IgniteStore<K, T extends PersistentBase> extends DataStoreBase<K, T
if (fieldSchema.getTypes().size() == 2 && isNullable(fieldSchema)) {
int schemaPos = getUnionSchema(fieldValue, fieldSchema);
Schema unionSchema = fieldSchema.getTypes().get(schemaPos);
- output = serializeFieldValue(get, unionSchema, fieldValue);
+ output = serializeFieldValue(unionSchema, fieldValue);
} else {
data = null;
try {
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
index 62cba67..49959c8 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/store/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
new file mode 100644
index 0000000..7459c67
--- /dev/null
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteBackendConstants.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.ignite.utils;
+
+/**
+ *
+ * Constants file for Ignite.
+ */
+public class IgniteBackendConstants {
+
+ /*
+ * Default configurations for Ignite
+ */
+ public static final String DEFAULT_IGNITE_HOST = "localhost";
+ public static final String DEFAULT_IGNITE_PORT = "10800";
+
+ /*
+ * A '42000' error code is thrown by Ignite when a non-existent table is queried.
+ * More details: https://apacheignite-sql.readme.io/docs/jdbc-error-codes
+ */
+ public static final String DEFAULT_IGNITE_TABLE_NOT_EXISTS_CODE = "42000";
+}
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
index 4c1e314..3639b90 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/IgniteSQLBuilder.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import org.apache.gora.ignite.store.Column;
import org.apache.gora.ignite.store.IgniteMapping;
@@ -40,10 +39,23 @@ public class IgniteSQLBuilder {
return messageFormat.format(args);
}
+ /**
+ * Returns a SQL query for determine whether a table exists or not.
+ *
+ * @param tableName The name of the table to be check.
+ * @return SQL query
+ */
public static String tableExists(String tableName) {
return format("SELECT * FROM {0} LIMIT 0", tableName);
}
+ /**
+ * Returns a SQL create table statement for initializing a datastore based
+ * upon a Ignite Mapping definition.
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @return SQL create query (DDL).
+ */
public static String createTable(IgniteMapping mapping) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("CREATE TABLE ");
@@ -67,10 +79,25 @@ public class IgniteSQLBuilder {
return sqlBuilder.toString();
}
+ /**
+ * Returns a SQL drop table statement for deleting a datastore instance within
+ * ignite.
+ *
+ * @param tableName The name of the table to be dropped.
+ * @return SQL drop query (DDL).
+ */
public static String dropTable(String tableName) {
return format("DROP TABLE IF EXISTS {0} ;", tableName);
}
+ /**
+ * Returns a bare SQL insert statement for adding a new record on a Ignite
+ * data store.
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @param data A map containing the Column-Value pairs of the new record.
+ * @return SQL insert statement
+ */
public static String baseInsertStatement(IgniteMapping mapping, Map<Column, Object> data) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("MERGE INTO ");
@@ -92,14 +119,31 @@ public class IgniteSQLBuilder {
return sqlBuilder.toString();
}
- public static void fillInsertStatement(PreparedStatement st, Map<Column, Object> data) throws SQLException {
- List<Entry<Column, Object>> list = new ArrayList<>(data.entrySet());
+ /**
+ * Fills a SQL PreparedStatement of a insert operation with the actual data to
+ * be inserted.
+ *
+ * @param statement The insertion PreparedStatement to be filled.
+ * @param insertData A map containing the Column-Value pairs of the new
+ * record.
+ * @throws SQLException When invalid values are provided as parameters for the
+ * insert statement.
+ */
+ public static void fillInsertStatement(PreparedStatement statement, Map<Column, Object> insertData) throws SQLException {
+ List<Entry<Column, Object>> list = new ArrayList<>(insertData.entrySet());
for (int i = 0; i < list.size(); i++) {
int j = i + 1;
- st.setObject(j, list.get(i).getValue());
+ statement.setObject(j, list.get(i).getValue());
}
}
+ /**
+ * Returns a bare SQL statement for deleting a record from the Ignite data
+ * store.
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @return SQL delete statement
+ */
public static String delete(IgniteMapping mapping) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("DELETE FROM ");
@@ -113,20 +157,38 @@ public class IgniteSQLBuilder {
return sqlBuilder.toString();
}
- public static void fillDeleteStatement(PreparedStatement st, IgniteMapping mapping, Object... data) throws SQLException {
- assert mapping.getPrimaryKey().size() == data.length;
+ /**
+ * Fills a SQL PreparedStatement of a delete operation with the actual key of
+ * the record to be deleted
+ *
+ * @param statement The deletion PreparedStatement to be filled.
+ * @param mapping The ignite mapping definition of the data store
+ * @param deleteData An Object array containing the primary key values of the
+ * record to be deleted
+ * @throws SQLException When invalid keys' values are provided as parameters
+ */
+ public static void fillDeleteStatement(PreparedStatement statement, IgniteMapping mapping, Object... deleteData) throws SQLException {
+ assert mapping.getPrimaryKey().size() == deleteData.length;
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
int j = i + 1;
- st.setObject(j, data[i]);
+ statement.setObject(j, deleteData[i]);
}
}
- public static String selectGet(IgniteMapping mapping, List<String> fields) {
+ /**
+ * Returns a bare SQL statement for retrieving a record from the ignite data
+ * store
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @param columns A list of columns to be retrieved within the select query
+ * @return SQL select statement
+ */
+ public static String selectGet(IgniteMapping mapping, List<String> columns) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("SELECT ");
- for (int i = 0; i < fields.size(); i++) {
- sqlBuilder.append(fields.get(i));
- sqlBuilder.append(i == fields.size() - 1 ? "" : " , ");
+ for (int i = 0; i < columns.size(); i++) {
+ sqlBuilder.append(columns.get(i));
+ sqlBuilder.append(i == columns.size() - 1 ? "" : " , ");
}
sqlBuilder.append(" FROM ");
sqlBuilder.append(mapping.getTableName());
@@ -139,20 +201,39 @@ public class IgniteSQLBuilder {
return sqlBuilder.toString();
}
- public static void fillSelectStatement(PreparedStatement st, IgniteMapping mapping, Object... data) throws SQLException {
- assert mapping.getPrimaryKey().size() == data.length;
+ /**
+ * Fills a SQL PreparedStatement of a select operation with the actual keys of
+ * the record to be retrieved
+ *
+ * @param statement The select PreparedStatement to be filled.
+ * @param mapping The ignite mapping definition of the data store
+ * @param selectData An Object array containing the primary key values of the
+ * record to be retrieved
+ * @throws SQLException When invalid keys' values are provided as parameters
+ */
+ public static void fillSelectStatement(PreparedStatement statement, IgniteMapping mapping, Object... selectData) throws SQLException {
+ assert mapping.getPrimaryKey().size() == selectData.length;
for (int i = 0; i < mapping.getPrimaryKey().size(); i++) {
int j = i + 1;
- st.setObject(j, data[i]);
+ statement.setObject(j, selectData[i]);
}
}
- public static String selectQuery(IgniteMapping mapping, List<String> ifields) {
+ /**
+ * Returns a base SQL statement for retrieving multiple records from the
+ * ignite data store
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @param selectFields A list of columns to be retrieved within the select
+ * query
+ * @return SQL select statement
+ */
+ public static String selectQuery(IgniteMapping mapping, List<String> selectFields) {
List<String> fields = new ArrayList<>();
for (Column c : mapping.getPrimaryKey()) {
fields.add(c.getName());
}
- fields.addAll(ifields);
+ fields.addAll(selectFields);
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("SELECT ");
for (int i = 0; i < fields.size(); i++) {
@@ -164,6 +245,13 @@ public class IgniteSQLBuilder {
return sqlBuilder.toString();
}
+ /**
+ * Returns a base SQL statement for deleting multiple records from the ignite
+ * data store
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @return SQL delete statement
+ */
public static String deleteQuery(IgniteMapping mapping) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("DELETE FROM ");
@@ -171,64 +259,91 @@ public class IgniteSQLBuilder {
return sqlBuilder.toString();
}
- public static String deleteQueryFields(IgniteMapping mapping, List<String> lsFields) {
+ /**
+ * Returns a base SQL statement for deleting fields from records of the ignite
+ * data store
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @param deleteFields A list of columns to be deleted (set to null)
+ * @return SQL update statement
+ */
+ public static String deleteQueryFields(IgniteMapping mapping, List<String> deleteFields) {
StringBuilder sqlBuilder = new StringBuilder();
sqlBuilder.append("UPDATE ");
sqlBuilder.append(mapping.getTableName());
- if (!lsFields.isEmpty()) {
+ if (!deleteFields.isEmpty()) {
sqlBuilder.append(" SET ");
}
- for (int i = 0; i < lsFields.size(); i++) {
- sqlBuilder.append(lsFields.get(i));
+ for (int i = 0; i < deleteFields.size(); i++) {
+ sqlBuilder.append(deleteFields.get(i));
sqlBuilder.append(" = null");
- sqlBuilder.append(i == lsFields.size() - 1 ? "" : " , ");
+ sqlBuilder.append(i == deleteFields.size() - 1 ? "" : " , ");
}
return sqlBuilder.toString();
}
- public static String selectQueryWhere(IgniteMapping mapping, Object sk, Object ek, long limit) {
+ /**
+ * Returns a SQL's WHERE segment with proper conditions set for
+ * Querying/Deleting/Updating multiple records of a ignite data store
+ *
+ * @param mapping The ignite mapping definition of the data store
+ * @param startKey Start key of the WHERE condition
+ * @param endKey End key of the WHERE condition
+ * @param limit The maximum number of records to be consider
+ * @return SQL WHERE segment
+ */
+ public static String queryWhere(IgniteMapping mapping, Object startKey, Object endKey, long limit) {
//composite keys pending
assert mapping.getPrimaryKey().size() == 1;
String keycolumn = mapping.getPrimaryKey().get(0).getName();
StringBuilder sqlBuilder = new StringBuilder();
- if (sk != null || ek != null) {
+ if (startKey != null || endKey != null) {
sqlBuilder.append(" WHERE ");
- if (sk != null && ek != null && sk.equals(ek)) {
+ if (startKey != null && endKey != null && startKey.equals(endKey)) {
sqlBuilder.append(keycolumn);
sqlBuilder.append("= ?");
} else {
- if (sk != null) {
+ if (startKey != null) {
sqlBuilder.append(keycolumn);
sqlBuilder.append(">= ?");
}
- if (sk != null && ek != null) {
+ if (startKey != null && endKey != null) {
sqlBuilder.append(" AND ");
}
- if (ek != null) {
+ if (endKey != null) {
sqlBuilder.append(keycolumn);
sqlBuilder.append("<= ?");
}
}
}
if (limit > 0) {
- sqlBuilder.append(" LIMIT " + limit);
+ sqlBuilder.append(" LIMIT ").append(limit);
}
return sqlBuilder.toString();
}
- public static void fillSelectQuery(PreparedStatement st, Object sk, Object ek) throws SQLException {
- if (sk != null || ek != null) {
- if (sk != null && ek != null && sk.equals(ek)) {
- st.setObject(1, sk);
+ /**
+ * Fills a SQL PreparedStatement's WHERE segment of a select/delete/update
+ * operation with proper key values
+ *
+ * @param statement The select PreparedStatement to be filled.
+ * @param startKey Start key of the WHERE condition
+ * @param endKey End key of the WHERE condition
+ * @throws SQLException When invalid keys' values are provided as parameters
+ */
+ public static void fillSelectQuery(PreparedStatement statement, Object startKey, Object endKey) throws SQLException {
+ if (startKey != null || endKey != null) {
+ if (startKey != null && endKey != null && startKey.equals(endKey)) {
+ statement.setObject(1, startKey);
} else {
- if (sk != null && ek != null) {
- st.setObject(1, sk);
- st.setObject(2, ek);
+ if (startKey != null && endKey != null) {
+ statement.setObject(1, startKey);
+ statement.setObject(2, endKey);
} else {
- if (sk != null) {
- st.setObject(1, sk);
+ if (startKey != null) {
+ statement.setObject(1, startKey);
} else {
- st.setObject(1, ek);
+ statement.setObject(1, endKey);
}
}
}
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
----------------------------------------------------------------------
diff --git a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
index ce4cd7a..ddba6e6 100644
--- a/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
+++ b/gora-ignite/src/main/java/org/apache/gora/ignite/utils/package-info.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/gora-ignite/src/test/resources/META-INF/config/java.util.logging.properties
----------------------------------------------------------------------
diff --git a/gora-ignite/src/test/resources/META-INF/config/java.util.logging.properties b/gora-ignite/src/test/resources/META-INF/config/java.util.logging.properties
new file mode 100644
index 0000000..72e267f
--- /dev/null
+++ b/gora-ignite/src/test/resources/META-INF/config/java.util.logging.properties
@@ -0,0 +1,74 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#########################################################################
+# Default java.util.logging configuration for Ignite.
+#
+# To use another config file use `java.util.logging.config.file` system
+# property. For example `java -Djava.util.logging.config.file=myfile`
+#########################################################################
+
+#
+# Comma-separated list of logging "handlers". Note that some of them may be
+# reconfigured (or even removed) at runtime according to system properties.
+#
+# By default all messages will be passed to console and file.
+#
+handlers=java.util.logging.ConsoleHandler, org.apache.ignite.logger.java.JavaLoggerFileHandler
+
+#
+# Default global logging level.
+# This specifies which kinds of events are logged across all loggers.
+# For any given category this global level can be overriden by a category
+# specific level.
+# Note that handlers also have a separate level setting to limit messages
+# printed through it.
+#
+.level=OFF
+
+#
+# Uncomment to allow debug messages for entire Ignite package.
+#
+#org.apache.ignite.level=FINE
+
+#
+# Uncomment this line to enable cache query execution tracing.
+#
+#org.apache.ignite.cache.queries.level=FINE
+
+#
+# Uncomment to disable courtesy notices, such as SPI configuration
+# consistency warnings.
+#
+#org.apache.ignite.CourtesyConfigNotice.level=OFF
+
+#
+# Console handler logs all messages with importance level `INFO` and above
+# into standard error stream (`System.err`).
+#
+java.util.logging.ConsoleHandler.formatter=org.apache.ignite.logger.java.JavaLoggerFormatter
+java.util.logging.ConsoleHandler.level=INFO
+
+#
+# File handler logs all messages into files with pattern `ignite-%{id8}.%g.log`
+# under `$IGNITE_HOME/work/log/` directory. The placeholder `%{id8}` is a truncated node ID.
+#
+org.apache.ignite.logger.java.JavaLoggerFileHandler.formatter=org.apache.ignite.logger.java.JavaLoggerFormatter
+org.apache.ignite.logger.java.JavaLoggerFileHandler.pattern=ignite-%{id8}.%g.log
+org.apache.ignite.logger.java.JavaLoggerFileHandler.level=INFO
+org.apache.ignite.logger.java.JavaLoggerFileHandler.limit=10485760
+org.apache.ignite.logger.java.JavaLoggerFileHandler.count=10
http://git-wip-us.apache.org/repos/asf/gora/blob/740ab814/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 88b9b52..b4838f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -780,6 +780,8 @@
<!-- Cassandra Dependencies -->
<cassandra-driver.version>3.3.0</cassandra-driver.version>
<cassandra.version>3.11.0</cassandra.version>
+ <!-- Ignite Dependencies -->
+ <ignite.version>2.6.0</ignite.version>
<!-- Solr Dependencies -->
<lucene-solr.version>6.5.1</lucene-solr.version>
<solr-solrj.version>6.5.1</solr-solrj.version>