You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/11/01 09:55:31 UTC
[01/19] ignite git commit: IGNITE-6641 Improved partition ID handling
for links - Fixes #2866.
Repository: ignite
Updated Branches:
refs/heads/ignite-6778 41ff5ff3c -> 98ca614b4
IGNITE-6641 Improved partition ID handling for links - Fixes #2866.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/007c579c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/007c579c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/007c579c
Branch: refs/heads/ignite-6778
Commit: 007c579c48e5361858ca5f5f77eae28034fe611b
Parents: 031f63c
Author: Sergey Chugunov <se...@gmail.com>
Authored: Fri Oct 27 18:31:49 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Oct 27 18:31:49 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/pagemem/PageIdUtils.java | 19 +++++++++++++++++++
.../cache/persistence/freelist/FreeListImpl.java | 18 ++++++++++++++++--
.../cache/persistence/tree/io/DataPageIO.java | 17 +++++++++++++++--
3 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/007c579c/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
index d47f2de..2754d79 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdUtils.java
@@ -191,6 +191,14 @@ public final class PageIdUtils {
}
/**
+ * Masks partition ID from full page ID.
+ * @param pageId Page ID to mask partition ID from.
+ */
+ public static long maskPartitionId(long pageId) {
+ return pageId & ~((-1L << PAGE_IDX_SIZE) & (~(-1L << PAGE_IDX_SIZE + PART_ID_SIZE)));
+ }
+
+ /**
* Change page type.
*
* @param pageId Old page ID.
@@ -213,4 +221,15 @@ public final class PageIdUtils {
")"
;
}
+
+ /**
+ * @param pageId Page ID.
+ * @param partId Partition ID.
+ */
+ public static long changePartitionId(long pageId, int partId) {
+ byte flag = flag(pageId);
+ int pageIdx = pageIndex(pageId);
+
+ return pageId(partId, flag, pageIdx);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/007c579c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
index 6a87d3e..5bacc53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
@@ -191,7 +191,7 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList {
CacheDataRow row,
int rowSize
) throws IgniteCheckedException {
- io.addRow(pageAddr, row, rowSize, pageSize());
+ io.addRow(pageId, pageAddr, row, rowSize, pageSize());
if (needWalDeltaRecord(pageId, page, null)) {
// TODO IGNITE-5829 This record must contain only a reference to a logical WAL record with the actual data.
@@ -256,12 +256,20 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList {
/** */
- private final PageHandler<Void, Long> rmvRow = new RemoveRowHandler();
+ private final PageHandler<Void, Long> rmvRow;
/**
*
*/
private final class RemoveRowHandler extends PageHandler<Void, Long> {
+ /** Indicates whether partition ID should be masked from page ID. */
+ private final boolean maskPartId;
+
+ /** */
+ RemoveRowHandler(boolean maskPartId) {
+ this.maskPartId = maskPartId;
+ }
+
@Override public Long run(
int cacheId,
long pageId,
@@ -293,6 +301,7 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList {
if (oldBucket != newBucket) {
// It is possible that page was concurrently taken for put, in this case put will handle bucket change.
+ pageId = maskPartId ? PageIdUtils.maskPartitionId(pageId) : pageId;
if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
put(null, pageId, page, pageAddr, newBucket);
}
@@ -330,6 +339,9 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList {
long metaPageId,
boolean initNew) throws IgniteCheckedException {
super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
+
+ rmvRow = new RemoveRowHandler(cacheId == 0);
+
this.evictionTracker = memPlc.evictionTracker();
this.reuseList = reuseList == null ? this : reuseList;
int pageSize = pageMem.pageSize();
@@ -492,6 +504,8 @@ public class FreeListImpl extends PagesList implements FreeList, ReuseList {
if (allocated)
pageId = allocateDataPage(row.partition());
+ else
+ pageId = PageIdUtils.changePartitionId(pageId, (row.partition()));
DataPageIO init = reuseBucket || allocated ? DataPageIO.VERSIONS.latest() : null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/007c579c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
index 628ff38..173ed66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
@@ -775,6 +775,7 @@ public class DataPageIO extends PageIO {
/**
* Adds row to this data page and sets respective link to the given row object.
*
+ * @param pageId page ID.
* @param pageAddr Page address.
* @param row Cache data row.
* @param rowSize Row size.
@@ -782,6 +783,7 @@ public class DataPageIO extends PageIO {
* @throws IgniteCheckedException If failed.
*/
public void addRow(
+ final long pageId,
final long pageAddr,
CacheDataRow row,
final int rowSize,
@@ -800,7 +802,7 @@ public class DataPageIO extends PageIO {
int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
- setLink(row, pageAddr, itemId);
+ setLinkByPageId(row, pageId, itemId);
}
/**
@@ -1021,7 +1023,18 @@ public class DataPageIO extends PageIO {
* @param itemId Item ID.
*/
private void setLink(CacheDataRow row, long pageAddr, int itemId) {
- row.link(PageIdUtils.link(getPageId(pageAddr), itemId));
+ long pageId = getPageId(pageAddr);
+
+ setLinkByPageId(row, pageId, itemId);
+ }
+
+ /**
+ * @param row Row to set link to.
+ * @param pageId Page ID.
+ * @param itemId Item ID.
+ */
+ private void setLinkByPageId(CacheDataRow row, long pageId, int itemId) {
+ row.link(PageIdUtils.link(pageId, itemId));
}
/**
[05/19] ignite git commit: IGNITE-6789 Web Console: Reworked sorting
of caches on Queries screen.
Posted by ag...@apache.org.
IGNITE-6789 Web Console: Reworked sorting of caches on Queries screen.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1d39507f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1d39507f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1d39507f
Branch: refs/heads/ignite-6778
Commit: 1d39507ff0051588885908aefa209829412a4f52
Parents: b4bd20e
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Mon Oct 30 19:21:15 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Mon Oct 30 19:21:15 2017 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/modules/sql/sql.controller.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1d39507f/modules/web-console/frontend/app/modules/sql/sql.controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/sql/sql.controller.js b/modules/web-console/frontend/app/modules/sql/sql.controller.js
index a3fc0ca..332d4d2 100644
--- a/modules/web-console/frontend/app/modules/sql/sql.controller.js
+++ b/modules/web-console/frontend/app/modules/sql/sql.controller.js
@@ -864,7 +864,7 @@ export default ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval',
});
return cachesAcc;
- }, []), 'label');
+ }, []), (cache) => cache.label.toLowerCase());
// Reset to first cache in case of stopped selected.
const cacheNames = _.map($scope.caches, (cache) => cache.value);
[07/19] ignite git commit: IGNITE-6398 Web Console: Added support for
ClientConnectorConfiguration.
Posted by ag...@apache.org.
IGNITE-6398 Web Console: Added support for ClientConnectorConfiguration.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/58dbbc53
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/58dbbc53
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/58dbbc53
Branch: refs/heads/ignite-6778
Commit: 58dbbc5394d603fabc348d92cda2a4e959fa0e79
Parents: 555b71e
Author: vsisko <vs...@gridgain.com>
Authored: Mon Oct 30 19:43:48 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Mon Oct 30 19:43:48 2017 +0700
----------------------------------------------------------------------
.../internal/visor/VisorDataTransferObject.java | 3 +
.../node/VisorClientConnectorConfiguration.java | 166 +++++++++++++++++++
.../visor/node/VisorGridConfiguration.java | 26 ++-
.../node/VisorSqlConnectorConfiguration.java | 3 +
.../config/VisorConfigurationCommand.scala | 29 +++-
modules/web-console/backend/app/mongo.js | 11 ++
.../generator/AbstractTransformer.js | 5 +
.../generator/ConfigurationGenerator.js | 35 +++-
.../configuration/clusters/client-connector.pug | 59 +++++++
.../configuration/clusters/sql-connector.pug | 4 +-
.../views/configuration/clusters.tpl.pug | 6 +
11 files changed, 332 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
index 8660891..4635c3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
@@ -41,6 +41,9 @@ public abstract class VisorDataTransferObject implements Externalizable {
/** Version 2. */
protected static final byte V2 = 2;
+ /** Version 3. */
+ protected static final byte V3 = 3;
+
/**
* @param col Source collection.
* @param <T> Collection type.
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorClientConnectorConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorClientConnectorConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorClientConnectorConfiguration.java
new file mode 100644
index 0000000..397b72a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorClientConnectorConfiguration.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.node;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Data transfer object for client connector configuration.
+ */
+public class VisorClientConnectorConfiguration extends VisorDataTransferObject {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Host. */
+ private String host;
+
+ /** Port. */
+ private int port;
+
+ /** Port range. */
+ private int portRange;
+
+ /** Max number of opened cursors per connection. */
+ private int maxOpenCursorsPerConn;
+
+ /** Socket send buffer size. */
+ private int sockSndBufSize;
+
+ /** Socket receive buffer size. */
+ private int sockRcvBufSize;
+
+ /** TCP no delay. */
+ private boolean tcpNoDelay;
+
+ /** Thread pool size. */
+ private int threadPoolSize ;
+
+ /**
+ * Default constructor.
+ */
+ public VisorClientConnectorConfiguration() {
+ // No-op.
+ }
+
+ /**
+ * Create data transfer object for Sql connector configuration.
+ *
+ * @param cfg Sql connector configuration.
+ */
+ public VisorClientConnectorConfiguration(ClientConnectorConfiguration cfg) {
+ host = cfg.getHost();
+ port = cfg.getPort();
+ portRange = cfg.getPortRange();
+ maxOpenCursorsPerConn = cfg.getMaxOpenCursorsPerConnection();
+ sockSndBufSize = cfg.getSocketSendBufferSize();
+ sockRcvBufSize = cfg.getSocketReceiveBufferSize();
+ tcpNoDelay = cfg.isTcpNoDelay();
+ threadPoolSize = cfg.getThreadPoolSize();
+ }
+
+ /**
+ * @return Host.
+ */
+ @Nullable public String getHost() {
+ return host;
+ }
+
+ /**
+ * @return Port.
+ */
+ public int getPort() {
+ return port;
+ }
+
+ /**
+ * @return Port range.
+ */
+ public int getPortRange() {
+ return portRange;
+ }
+
+ /**
+ * @return Maximum number of opened cursors.
+ */
+ public int getMaxOpenCursorsPerConnection() {
+ return maxOpenCursorsPerConn;
+ }
+
+ /**
+ * @return Socket send buffer size in bytes.
+ */
+ public int getSocketSendBufferSize() {
+ return sockSndBufSize;
+ }
+
+ /**
+ * @return Socket receive buffer size in bytes.
+ */
+ public int getSocketReceiveBufferSize() {
+ return sockRcvBufSize;
+ }
+
+ /**
+ * @return TCP NO_DELAY flag.
+ */
+ public boolean isTcpNoDelay() {
+ return tcpNoDelay;
+ }
+ /**
+ * @return Thread pool that is in charge of processing SQL requests.
+ */
+ public int getThreadPoolSize() {
+ return threadPoolSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+ U.writeString(out, host);
+ out.writeInt(port);
+ out.writeInt(portRange);
+ out.writeInt(maxOpenCursorsPerConn);
+ out.writeInt(sockSndBufSize);
+ out.writeInt(sockRcvBufSize );
+ out.writeBoolean(tcpNoDelay);
+ out.writeInt(threadPoolSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+ host = U.readString(in);
+ port = in.readInt();
+ portRange = in.readInt();
+ maxOpenCursorsPerConn = in.readInt();
+ sockSndBufSize = in.readInt();
+ sockRcvBufSize = in.readInt();
+ tcpNoDelay = in.readBoolean();
+ threadPoolSize = in.readInt();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(VisorClientConnectorConfiguration.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
index 99cce40..9e2370c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorGridConfiguration.java
@@ -26,9 +26,9 @@ import java.util.Map;
import java.util.Properties;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
import org.apache.ignite.configuration.HadoopConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.SqlConnectorConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -122,6 +122,9 @@ public class VisorGridConfiguration extends VisorDataTransferObject {
/** Configuration of data storage. */
private VisorDataStorageConfiguration dataStorage;
+ /** Client connector configuration */
+ private VisorClientConnectorConfiguration clnConnCfg;
+
/**
* Default constructor.
*/
@@ -177,10 +180,10 @@ public class VisorGridConfiguration extends VisorDataTransferObject {
if (hc != null)
hadoopCfg = new VisorHadoopConfiguration(hc);
- SqlConnectorConfiguration scc = c.getSqlConnectorConfiguration();
+ ClientConnectorConfiguration ccc = c.getClientConnectorConfiguration();
- if (scc != null)
- sqlConnCfg = new VisorSqlConnectorConfiguration(scc);
+ if (ccc != null)
+ clnConnCfg = new VisorClientConnectorConfiguration(ccc);
srvcCfgs = VisorServiceConfiguration.list(c.getServiceConfiguration());
@@ -356,6 +359,13 @@ public class VisorGridConfiguration extends VisorDataTransferObject {
}
/**
+ * @return Client connector configuration.
+ */
+ public VisorClientConnectorConfiguration getClientConnectorConfiguration() {
+ return clnConnCfg;
+ }
+
+ /**
* @return List of service configurations
*/
public List<VisorServiceConfiguration> getServiceConfigurations() {
@@ -371,7 +381,7 @@ public class VisorGridConfiguration extends VisorDataTransferObject {
/** {@inheritDoc} */
@Override public byte getProtocolVersion() {
- return V2;
+ return V3;
}
/** {@inheritDoc} */
@@ -402,6 +412,7 @@ public class VisorGridConfiguration extends VisorDataTransferObject {
out.writeObject(sqlConnCfg);
U.writeCollection(out, srvcCfgs);
out.writeObject(dataStorage);
+ out.writeObject(clnConnCfg);
}
/** {@inheritDoc} */
@@ -432,8 +443,11 @@ public class VisorGridConfiguration extends VisorDataTransferObject {
sqlConnCfg = (VisorSqlConnectorConfiguration) in.readObject();
srvcCfgs = U.readList(in);
- if (protoVer == V2)
+ if (protoVer >= V2)
dataStorage = (VisorDataStorageConfiguration)in.readObject();
+
+ if (protoVer >= V3)
+ clnConnCfg = (VisorClientConnectorConfiguration)in.readObject();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorSqlConnectorConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorSqlConnectorConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorSqlConnectorConfiguration.java
index ccde415..2c2f959 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorSqlConnectorConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorSqlConnectorConfiguration.java
@@ -28,7 +28,10 @@ import org.jetbrains.annotations.Nullable;
/**
* Data transfer object for SQL connector configuration.
+ *
+ * Deprecated as of Apache Ignite 2.3
*/
+@Deprecated
public class VisorSqlConnectorConfiguration extends VisorDataTransferObject {
/** */
private static final long serialVersionUID = 0L;
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/config/VisorConfigurationCommand.scala
----------------------------------------------------------------------
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/config/VisorConfigurationCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/config/VisorConfigurationCommand.scala
index 13ab778..b9b0af9 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/config/VisorConfigurationCommand.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/config/VisorConfigurationCommand.scala
@@ -17,19 +17,16 @@
package org.apache.ignite.visor.commands.config
+import java.util.UUID
import org.apache.ignite.cluster.ClusterGroupEmptyException
import org.apache.ignite.internal.util.scala.impl
import org.apache.ignite.internal.util.{IgniteUtils => U}
-import org.apache.ignite.lang.IgniteBiTuple
import org.apache.ignite.visor.VisorTag
import org.apache.ignite.visor.commands.cache.VisorCacheCommand._
import org.apache.ignite.visor.commands.common.{VisorConsoleCommand, VisorTextTable}
import org.apache.ignite.visor.visor._
-import java.lang.System._
-import java.util.UUID
-
-import org.apache.ignite.internal.visor.node.{VisorSpiDescription, VisorGridConfiguration, VisorNodeConfigurationCollectorTask}
+import org.apache.ignite.internal.visor.node.{VisorGridConfiguration, VisorNodeConfigurationCollectorTask, VisorSpiDescription}
import org.apache.ignite.internal.visor.util.VisorTaskUtils._
import scala.collection.JavaConversions._
@@ -240,6 +237,26 @@ class VisorConfigurationCommand extends VisorConsoleCommand {
spisT.render()
+ println("\nClient connector configuration")
+
+ val cliConnCfg = cfg.getClientConnectorConfiguration
+ val cliConnTbl = VisorTextTable()
+
+ if (cliConnCfg != null) {
+ cliConnTbl += ("Host", safe(cliConnCfg.getHost, safe(basic.getLocalHost)))
+ cliConnTbl += ("Port", cliConnCfg.getPort)
+ cliConnTbl += ("Port range", cliConnCfg.getPortRange)
+ cliConnTbl += ("Socket send buffer size", formatMemory(cliConnCfg.getSocketSendBufferSize))
+ cliConnTbl += ("Socket receive buffer size", formatMemory(cliConnCfg.getSocketReceiveBufferSize))
+ cliConnTbl += ("Max connection cursors", cliConnCfg.getMaxOpenCursorsPerConnection)
+ cliConnTbl += ("Pool size", cliConnCfg.getThreadPoolSize)
+ cliConnTbl += ("TCP_NODELAY", bool2Str(cliConnCfg.isTcpNoDelay))
+
+ cliConnTbl.render()
+ }
+ else
+ println("Client Connection is not configured")
+
println("\nPeer-to-Peer:")
val p2pT = VisorTextTable()
@@ -371,7 +388,7 @@ class VisorConfigurationCommand extends VisorConsoleCommand {
* @return List of strings.
*/
private[this] def compactProperty(name: String, value: String): List[String] = {
- val ps = getProperty("path.separator")
+ val ps = System.getProperty("path.separator")
// Split all values having path separator into multiple lines (with few exceptions...).
val lst =
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/web-console/backend/app/mongo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 5b02a72..7043fcd 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -854,6 +854,17 @@ module.exports.factory = function(passportMongo, settings, pluginMongo, mongoose
className: String
}
}],
+ clientConnectorConfiguration: {
+ enabled: Boolean,
+ host: String,
+ port: Number,
+ portRange: Number,
+ socketSendBufferSize: Number,
+ socketReceiveBufferSize: Number,
+ tcpNoDelay: {type: Boolean, default: true},
+ maxOpenCursorsPerConnection: Number,
+ threadPoolSize: Number
+ },
loadBalancingSpi: [{
kind: {type: String, enum: ['RoundRobin', 'Adaptive', 'WeightedRandom', 'Custom']},
RoundRobin: {
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
index f6f471c..af799bd 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
@@ -78,6 +78,11 @@ export default class AbstractTransformer {
return this.toSection(this.generator.clusterCacheKeyConfiguration(keyCfgs));
}
+ // Generate client connector configuration.
+ static clusterClientConnector(cluster, available) {
+ return this.toSection(this.generator.clusterClientConnector(cluster, available));
+ }
+
// Generate collision group.
static clusterCollision(collision) {
return this.toSection(this.generator.clusterCollision(collision));
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 1b12d52..16202f8 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -78,6 +78,10 @@ export default class IgniteConfigurationGenerator {
this.clusterBinary(cluster.binaryConfiguration, cfg);
this.clusterCacheKeyConfiguration(cluster.cacheKeyConfiguration, cfg);
this.clusterCheckpoint(cluster, cluster.caches, cfg);
+
+ if (available('2.3.0'))
+ this.clusterClientConnector(cluster, available, cfg);
+
this.clusterCollision(cluster.collision, cfg);
this.clusterCommunication(cluster, cfg);
this.clusterConnector(cluster.connector, cfg);
@@ -106,7 +110,9 @@ export default class IgniteConfigurationGenerator {
if (available(['2.1.0', '2.3.0']))
this.clusterPersistence(cluster.persistenceStoreConfiguration, available, cfg);
- this.clusterQuery(cluster, available, cfg);
+ if (available(['2.1.0', '2.3.0']))
+ this.clusterQuery(cluster, available, cfg);
+
this.clusterServiceConfiguration(cluster.serviceConfigurations, cluster.caches, cfg);
this.clusterSsl(cluster, cfg);
@@ -760,6 +766,33 @@ export default class IgniteConfigurationGenerator {
return cfg;
}
+ // Generate cluster query group.
+ static clusterClientConnector(cluster, available, cfg = this.igniteConfigurationBean(cluster)) {
+ if (!available('2.3.0'))
+ return cfg;
+
+ cfg.intProperty('longQueryWarningTimeout');
+
+ if (_.get(cluster, 'clientConnectorConfiguration.enabled') !== true)
+ return cfg;
+
+ const bean = new Bean('org.apache.ignite.configuration.ClientConnectorConfiguration', 'cliConnCfg',
+ cluster.clientConnectorConfiguration, clusterDflts.clientConnectorConfiguration);
+
+ bean.stringProperty('host')
+ .intProperty('port')
+ .intProperty('portRange')
+ .intProperty('socketSendBufferSize')
+ .intProperty('socketReceiveBufferSize')
+ .intProperty('maxOpenCursorsPerConnection')
+ .intProperty('threadPoolSize')
+ .boolProperty('tcpNoDelay');
+
+ cfg.beanProperty('clientConnectorConfiguration', bean);
+
+ return cfg;
+ }
+
// Generate collision group.
static clusterCollision(collision, cfg = this.igniteConfigurationBean()) {
let colSpi;
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/web-console/frontend/app/modules/states/configuration/clusters/client-connector.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/client-connector.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/client-connector.pug
new file mode 100644
index 0000000..dd2fa6d
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/client-connector.pug
@@ -0,0 +1,59 @@
+//-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+include /app/helpers/jade/mixins
+
+-var form = 'clientConnector'
+-var model = 'backupItem'
+-var connectionModel = model + '.clientConnectorConfiguration'
+-var connectionEnabled = connectionModel + '.enabled'
+
+.panel.panel-default(ng-show='$ctrl.available("2.3.0")' ng-form=form novalidate)
+ .panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
+ ignite-form-panel-chevron
+ label Client connector configuration
+ ignite-form-field-tooltip.tipLabel
+ | Client connector configuration
+ ignite-form-revert
+ .panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
+ .panel-body(ng-if=`$ctrl.available("2.3.0") && ui.isPanelLoaded('${form}')`)
+ .col-sm-6
+ .settings-row
+ +checkbox('Enabled', connectionEnabled, '"ClientConnectorEnabled"', 'Flag indicating whether to configure client connector configuration')
+ .settings-row
+ +text-enabled('Host:', `${connectionModel}.host`, '"ClientConnectorHost"', connectionEnabled, 'false', 'localhost', 'Host')
+ .settings-row
+ +number('Port:', `${connectionModel}.port`, '"ClientConnectorPort"', connectionEnabled, '10800', '1025', 'Port')
+ .settings-row
+ +number('Port range:', `${connectionModel}.portRange`, '"ClientConnectorPortRange"', connectionEnabled, '100', '0', 'Port range')
+ .settings-row
+ +number('Socket send buffer size:', `${connectionModel}.socketSendBufferSize`, '"ClientConnectorSocketSendBufferSize"', connectionEnabled, '0', '0',
+ 'Socket send buffer size.<br/>\
+ When set to <b>0</b>, operation system default will be used')
+ .settings-row
+ +number('Socket receive buffer size:', `${connectionModel}.socketReceiveBufferSize`, '"ClientConnectorSocketReceiveBufferSize"', connectionEnabled, '0', '0',
+ 'Socket receive buffer size.<br/>\
+ When set to <b>0</b>, operation system default will be used')
+ .settings-row
+ +number('Max connection cursors:', `${connectionModel}.maxOpenCursorsPerConnection`, '"ClientConnectorMaxOpenCursorsPerConnection"', connectionEnabled, '128', '0',
+ 'Max number of opened cursors per connection')
+ .settings-row
+ +number('Pool size:', `${connectionModel}.threadPoolSize`, '"ClientConnectorThreadPoolSize"', connectionEnabled, 'max(8, availableProcessors)', '1',
+ 'Size of thread pool that is in charge of processing SQL requests')
+ .settings-row
+ +checkbox-enabled('TCP_NODELAY option', `${connectionModel}.tcpNoDelay`, '"ClientConnectorTcpNoDelay"', connectionEnabled, 'Value for TCP_NODELAY socket option')
+ .col-sm-6
+ +preview-xml-java(model, 'clusterClientConnector')
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug
index 89e00d7..d72f962 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug
@@ -21,7 +21,7 @@ include /app/helpers/jade/mixins
-var connectionModel = model + '.sqlConnectorConfiguration'
-var connectionEnabled = connectionModel + '.enabled'
-.panel.panel-default(ng-show='$ctrl.available("2.1.0")' ng-form=form novalidate)
+.panel.panel-default(ng-show='$ctrl.available(["2.1.0", "2.3.0"])' ng-form=form novalidate)
.panel-heading(bs-collapse-toggle ng-click=`ui.loadPanel('${form}')`)
ignite-form-panel-chevron
label Query configuration
@@ -30,7 +30,7 @@ include /app/helpers/jade/mixins
//- TODO IGNITE-5415 Add link to documentation.
ignite-form-revert
.panel-collapse(role='tabpanel' bs-collapse-target id=`${form}`)
- .panel-body(ng-if=`$ctrl.available("2.1.0") && ui.isPanelLoaded('${form}')`)
+ .panel-body(ng-if=`$ctrl.available(["2.1.0", "2.3.0"]) && ui.isPanelLoaded('${form}')`)
.col-sm-6
.settings-row
+checkbox('Enabled', connectionEnabled, '"SqlConnectorEnabled"', 'Flag indicating whether to configure SQL connector configuration')
http://git-wip-us.apache.org/repos/asf/ignite/blob/58dbbc53/modules/web-console/frontend/views/configuration/clusters.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/configuration/clusters.tpl.pug b/modules/web-console/frontend/views/configuration/clusters.tpl.pug
index 26a1da1..19ed350 100644
--- a/modules/web-console/frontend/views/configuration/clusters.tpl.pug
+++ b/modules/web-console/frontend/views/configuration/clusters.tpl.pug
@@ -45,6 +45,10 @@ include /app/helpers/jade/mixins
include /app/modules/states/configuration/clusters/binary
include /app/modules/states/configuration/clusters/cache-key-cfg
include /app/modules/states/configuration/clusters/checkpoint
+
+ //- Since ignite 2.3
+ include /app/modules/states/configuration/clusters/client-connector
+
include /app/modules/states/configuration/clusters/collision
include /app/modules/states/configuration/clusters/communication
include /app/modules/states/configuration/clusters/connector
@@ -73,6 +77,8 @@ include /app/helpers/jade/mixins
//- Since ignite 2.1, deprecated in ignite 2.3
include /app/modules/states/configuration/clusters/persistence
+
+ //- Deprecated in ignite 2.3
include /app/modules/states/configuration/clusters/sql-connector
include /app/modules/states/configuration/clusters/service
[04/19] ignite git commit: IGNITE-6670 Web Agent: Improved demo
startup.
Posted by ag...@apache.org.
IGNITE-6670 Web Agent: Improved demo startup.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b4bd20e1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b4bd20e1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b4bd20e1
Branch: refs/heads/ignite-6778
Commit: b4bd20e10cf564c9852be58e620f1d3b79af9067
Parents: 67859f4
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Mon Oct 30 14:56:22 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Mon Oct 30 14:56:22 2017 +0700
----------------------------------------------------------------------
.../ignite/console/demo/AgentClusterDemo.java | 32 +++++++++++---------
.../demo/service/DemoCachesLoadService.java | 3 +-
2 files changed, 19 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b4bd20e1/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
index 886888b..7157df7 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
@@ -27,9 +27,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteServices;
import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.console.demo.service.DemoCachesLoadService;
import org.apache.ignite.console.demo.service.DemoComputeLoadService;
import org.apache.ignite.console.demo.service.DemoRandomCacheLoadService;
@@ -53,6 +53,7 @@ import static org.apache.ignite.IgniteSystemProperties.IGNITE_NO_ASCII;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PERFORMANCE_SUGGESTIONS_DISABLED;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_QUIET;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_UPDATE_NOTIFIER;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE;
import static org.apache.ignite.console.demo.AgentDemoUtils.newScheduledThreadPool;
import static org.apache.ignite.events.EventType.EVTS_DISCOVERY;
import static org.apache.ignite.internal.visor.util.VisorTaskUtils.VISOR_TASK_EVTS;
@@ -132,15 +133,16 @@ public class AgentClusterDemo {
cfg.setGridLogger(new Slf4jLogger(log));
cfg.setMetricsLogFrequency(0);
- MemoryConfiguration memCfg = new MemoryConfiguration();
+ DataRegionConfiguration dataRegCfg = new DataRegionConfiguration();
+ dataRegCfg.setName("demo");
+ dataRegCfg.setMetricsEnabled(true);
+ dataRegCfg.setMaxSize(DFLT_DATA_REGION_INITIAL_SIZE);
- MemoryPolicyConfiguration memPlc = new MemoryPolicyConfiguration();
- memPlc.setName("demo");
- memPlc.setMetricsEnabled(true);
+ DataStorageConfiguration dataStorageCfg = new DataStorageConfiguration();
+ dataStorageCfg.setDefaultDataRegionConfiguration(dataRegCfg);
+ dataStorageCfg.setSystemRegionMaxSize(DFLT_DATA_REGION_INITIAL_SIZE);
- memCfg.setMemoryPolicies(memPlc);
-
- cfg.setMemoryConfiguration(memCfg);
+ cfg.setDataStorageConfiguration(dataStorageCfg);
if (client)
cfg.setClientMode(true);
@@ -157,10 +159,10 @@ public class AgentClusterDemo {
services.deployMultiple("Demo service: Multiple instances", new DemoServiceMultipleInstances(), 7, 3);
services.deployNodeSingleton("Demo service: Node singleton", new DemoServiceNodeSingleton());
services.deployClusterSingleton("Demo service: Cluster singleton", new DemoServiceClusterSingleton());
+ services.deployClusterSingleton("Demo caches load service", new DemoCachesLoadService(20));
services.deployKeyAffinitySingleton("Demo service: Key affinity singleton",
new DemoServiceKeyAffinity(), DemoCachesLoadService.CAR_CACHE_NAME, "id");
- services.deployClusterSingleton("Demo caches load service", new DemoCachesLoadService(20));
services.deployNodeSingleton("RandomCache load service", new DemoRandomCacheLoadService(20));
services.deployMultiple("Demo service: Compute load", new DemoComputeLoadService(), 2, 1);
@@ -195,8 +197,10 @@ public class AgentClusterDemo {
int idx = cnt.incrementAndGet();
int port = basePort.get();
+ IgniteEx ignite = null;
+
try {
- IgniteEx ignite = (IgniteEx)Ignition.start(igniteConfiguration(port, idx, false));
+ ignite = (IgniteEx)Ignition.start(igniteConfiguration(port, idx, false));
if (idx == 0) {
Collection<String> jettyAddrs = ignite.localNode().attribute(ATTR_REST_JETTY_ADDRS);
@@ -219,8 +223,6 @@ public class AgentClusterDemo {
demoUrl = String.format("http://%s:%d", jettyHost, jettyPort);
initLatch.countDown();
-
- deployServices(ignite.services(ignite.cluster().forServers()));
}
}
catch (Throwable e) {
@@ -234,13 +236,15 @@ public class AgentClusterDemo {
}
finally {
if (idx == NODE_CNT) {
+ deployServices(ignite.services(ignite.cluster().forServers()));
+
log.info("DEMO: All embedded nodes for demo successfully started");
execSrv.shutdown();
}
}
}
- }, 1, 10, TimeUnit.SECONDS);
+ }, 1, 5, TimeUnit.SECONDS);
}
return initLatch;
http://git-wip-us.apache.org/repos/asf/ignite/blob/b4bd20e1/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index 40fd4ac..6691d1d 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -194,7 +194,6 @@ public class DemoCachesLoadService implements Service {
}, 10, 3, TimeUnit.SECONDS);
}
-
/**
* Create base cache configuration.
*
@@ -208,7 +207,7 @@ public class DemoCachesLoadService implements Service {
ccfg.setQueryDetailMetricsSize(10);
ccfg.setStatisticsEnabled(true);
ccfg.setSqlFunctionClasses(SQLFunctions.class);
- ccfg.setMemoryPolicyName("demo");
+ ccfg.setDataRegionName("demo");
return ccfg;
}
[03/19] ignite git commit: IGNITE-6570 Added missing import.
Posted by ag...@apache.org.
IGNITE-6570 Added missing import.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/67859f4b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/67859f4b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/67859f4b
Branch: refs/heads/ignite-6778
Commit: 67859f4b65ac4fb28b6b6199c0c7033b107c6de9
Parents: 24bd916
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Mon Oct 30 11:07:21 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Mon Oct 30 11:07:21 2017 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/utils/SimpleWorkerPool.js | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/67859f4b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
index d8ed28b..b76dc52 100644
--- a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
+++ b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
@@ -19,6 +19,7 @@ import {Observable} from 'rxjs/Observable';
import {Subject} from 'rxjs/Subject';
import 'rxjs/add/observable/race';
import 'rxjs/add/operator/filter';
+import 'rxjs/add/operator/map';
import 'rxjs/add/operator/pluck';
import 'rxjs/add/operator/take';
import 'rxjs/add/operator/toPromise';
[15/19] ignite git commit: Added docker file for 2.3.0 version.
Posted by ag...@apache.org.
Added docker file for 2.3.0 version.
Signed-off-by: nikolay_tikhonov <nt...@gridgain.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/67ca745b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/67ca745b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/67ca745b
Branch: refs/heads/ignite-6778
Commit: 67ca745b034d62b5e6666f9799d8235b9d5986bd
Parents: 94dd19d
Author: nikolay_tikhonov <nt...@gridgain.com>
Authored: Tue Oct 31 19:01:15 2017 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Tue Oct 31 19:01:15 2017 +0300
----------------------------------------------------------------------
modules/docker/2.3.0/Dockerfile | 46 ++++++++++++++++++++++++++++++++
modules/docker/2.3.0/run.sh | 51 ++++++++++++++++++++++++++++++++++++
modules/docker/Dockerfile | 2 +-
3 files changed, 98 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/67ca745b/modules/docker/2.3.0/Dockerfile
----------------------------------------------------------------------
diff --git a/modules/docker/2.3.0/Dockerfile b/modules/docker/2.3.0/Dockerfile
new file mode 100644
index 0000000..88ec8ba
--- /dev/null
+++ b/modules/docker/2.3.0/Dockerfile
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Start from a Java image.
+FROM openjdk:8
+
+# Ignite version
+ENV IGNITE_VERSION 2.3.0
+
+# Ignite home
+ENV IGNITE_HOME /opt/ignite/apache-ignite-fabric-${IGNITE_VERSION}-bin
+
+# Do not rely on anything provided by base image(s), but be explicit, if they are installed already it is noop then
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ unzip \
+ curl \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/ignite
+
+RUN curl https://dist.apache.org/repos/dist/release/ignite/${IGNITE_VERSION}/apache-ignite-fabric-${IGNITE_VERSION}-bin.zip -o ignite.zip \
+ && unzip ignite.zip \
+ && rm ignite.zip
+
+# Copy sh files and set permission
+COPY ./run.sh $IGNITE_HOME/
+
+RUN chmod +x $IGNITE_HOME/run.sh
+
+CMD $IGNITE_HOME/run.sh
+
+EXPOSE 11211 47100 47500 49112
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/67ca745b/modules/docker/2.3.0/run.sh
----------------------------------------------------------------------
diff --git a/modules/docker/2.3.0/run.sh b/modules/docker/2.3.0/run.sh
new file mode 100644
index 0000000..3aafc30
--- /dev/null
+++ b/modules/docker/2.3.0/run.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ ! -z "$OPTION_LIBS" ]; then
+ IFS=, LIBS_LIST=("$OPTION_LIBS")
+
+ for lib in ${LIBS_LIST[@]}; do
+ cp -r $IGNITE_HOME/libs/optional/"$lib"/* \
+ $IGNITE_HOME/libs/
+ done
+fi
+
+if [ ! -z "$EXTERNAL_LIBS" ]; then
+ IFS=, LIBS_LIST=("$EXTERNAL_LIBS")
+
+ for lib in ${LIBS_LIST[@]}; do
+ echo $lib >> temp
+ done
+
+ wget -i temp -P $IGNITE_HOME/libs
+
+ rm temp
+fi
+
+QUIET=""
+
+if [ "$IGNITE_QUIET" = "false" ]; then
+ QUIET="-v"
+fi
+
+if [ -z $CONFIG_URI ]; then
+ $IGNITE_HOME/bin/ignite.sh $QUIET
+else
+ $IGNITE_HOME/bin/ignite.sh $QUIET $CONFIG_URI
+fi
+
http://git-wip-us.apache.org/repos/asf/ignite/blob/67ca745b/modules/docker/Dockerfile
----------------------------------------------------------------------
diff --git a/modules/docker/Dockerfile b/modules/docker/Dockerfile
index 4c09653..88ec8ba 100644
--- a/modules/docker/Dockerfile
+++ b/modules/docker/Dockerfile
@@ -19,7 +19,7 @@
FROM openjdk:8
# Ignite version
-ENV IGNITE_VERSION 2.2.0
+ENV IGNITE_VERSION 2.3.0
# Ignite home
ENV IGNITE_HOME /opt/ignite/apache-ignite-fabric-${IGNITE_VERSION}-bin
[02/19] ignite git commit: IGNITE-6774 Java doc is broken:
"LUDecomposition.java:40: warning - Tag @see: missing final '>'". This closes
#2941.
Posted by ag...@apache.org.
IGNITE-6774 Java doc is broken: "LUDecomposition.java:40: warning - Tag @see: missing final '>'". This closes #2941.
Signed-off-by: nikolay_tikhonov <nt...@gridgain.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/24bd9167
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/24bd9167
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/24bd9167
Branch: refs/heads/ignite-6778
Commit: 24bd916742c8a84d2965324c4c9ea8b8571bfe62
Parents: 007c579
Author: mcherkasov <mi...@gmail.com>
Authored: Fri Oct 27 19:55:10 2017 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Fri Oct 27 19:55:10 2017 +0300
----------------------------------------------------------------------
.../org/apache/ignite/ml/math/decompositions/LUDecomposition.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/24bd9167/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java
index df7b6cd..8b79d9b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/decompositions/LUDecomposition.java
@@ -35,7 +35,7 @@ import static org.apache.ignite.ml.math.util.MatrixUtil.likeVector;
* @see <a href="http://mathworld.wolfram.com/LUDecomposition.html">MathWorld</a>
* @see <a href="http://en.wikipedia.org/wiki/LU_decomposition">Wikipedia</a>
*
- * TODO: IGNITE-5828, Maybe we should make this class (and other decompositions) Externalizable.
+ * <p>TODO: IGNITE-5828, Maybe we should make this class (and other decompositions) Externalizable.</p>
*/
public class LUDecomposition implements Destroyable {
/** Default bound to determine effective singularity in LU decomposition. */
[12/19] ignite git commit: IGNITE-6357: Added support of multiple SQL
statements for ODBC
Posted by ag...@apache.org.
IGNITE-6357: Added support of multiple SQL statements for ODBC
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/87e37975
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/87e37975
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/87e37975
Branch: refs/heads/ignite-6778
Commit: 87e37975d2cd24e2435de627d3ad6d8544af5972
Parents: 61621b8
Author: Igor Sapego <is...@gridgain.com>
Authored: Tue Oct 31 14:20:45 2017 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Tue Oct 31 14:20:45 2017 +0300
----------------------------------------------------------------------
.../processors/odbc/odbc/OdbcMessageParser.java | 60 +++++-
.../odbc/odbc/OdbcQueryExecuteBatchResult.java | 20 +-
.../odbc/odbc/OdbcQueryExecuteResult.java | 11 +-
.../odbc/odbc/OdbcQueryMoreResultsRequest.java | 61 ++++++
.../odbc/odbc/OdbcQueryMoreResultsResult.java | 66 ++++++
.../processors/odbc/odbc/OdbcQueryResults.java | 106 ++++++++++
.../processors/odbc/odbc/OdbcRequest.java | 3 +
.../odbc/odbc/OdbcRequestHandler.java | 175 +++++++---------
.../processors/odbc/odbc/OdbcResultSet.java | 101 ++++++++++
.../processors/odbc/odbc/OdbcUtils.java | 31 +++
.../examples/odbc-example/src/odbc_example.cpp | 8 +-
.../cpp/odbc-test/src/queries_test.cpp | 200 ++++++++++++++++++-
.../cpp/odbc/include/ignite/odbc/message.h | 97 ++++++++-
.../include/ignite/odbc/query/batch_query.h | 16 +-
.../ignite/odbc/query/column_metadata_query.h | 7 +
.../odbc/include/ignite/odbc/query/data_query.h | 22 +-
.../ignite/odbc/query/foreign_keys_query.h | 7 +
.../ignite/odbc/query/primary_keys_query.h | 7 +
.../cpp/odbc/include/ignite/odbc/query/query.h | 7 +
.../ignite/odbc/query/special_columns_query.h | 7 +
.../ignite/odbc/query/table_metadata_query.h | 7 +
.../include/ignite/odbc/query/type_info_query.h | 7 +
.../cpp/odbc/include/ignite/odbc/statement.h | 6 +-
modules/platforms/cpp/odbc/src/cursor.cpp | 2 +-
modules/platforms/cpp/odbc/src/message.cpp | 62 +++++-
modules/platforms/cpp/odbc/src/odbc.cpp | 2 +-
.../cpp/odbc/src/query/batch_query.cpp | 80 ++++----
.../odbc/src/query/column_metadata_query.cpp | 5 +
.../platforms/cpp/odbc/src/query/data_query.cpp | 93 ++++++++-
.../cpp/odbc/src/query/foreign_keys_query.cpp | 5 +
.../cpp/odbc/src/query/primary_keys_query.cpp | 5 +
.../odbc/src/query/special_columns_query.cpp | 5 +
.../cpp/odbc/src/query/table_metadata_query.cpp | 5 +
.../cpp/odbc/src/query/type_info_query.cpp | 5 +
modules/platforms/cpp/odbc/src/statement.cpp | 14 +-
35 files changed, 1103 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
index 04e2e25..bf74bc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
@@ -165,6 +165,15 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
break;
}
+ case OdbcRequest.MORE_RESULTS: {
+ long queryId = reader.readLong();
+ int pageSize = reader.readInt();
+
+ res = new OdbcQueryMoreResultsRequest(queryId, pageSize);
+
+ break;
+ }
+
default:
throw new IgniteException("Unknown ODBC command: [cmd=" + cmd + ']');
}
@@ -233,13 +242,13 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
for (OdbcColumnMeta meta : metas)
meta.write(writer);
- writer.writeLong(res.affectedRows());
+ writeAffectedRows(writer, res.affectedRows());
}
else if (res0 instanceof OdbcQueryExecuteBatchResult) {
OdbcQueryExecuteBatchResult res = (OdbcQueryExecuteBatchResult) res0;
writer.writeBoolean(res.errorMessage() == null);
- writer.writeLong(res.rowsAffected());
+ writeAffectedRows(writer, res.affectedRows());
if (res.errorMessage() != null) {
writer.writeLong(res.errorSetIdx());
@@ -276,6 +285,33 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
}
}
}
+ else if (res0 instanceof OdbcQueryMoreResultsResult) {
+ OdbcQueryMoreResultsResult res = (OdbcQueryMoreResultsResult) res0;
+
+ if (log.isDebugEnabled())
+ log.debug("Resulting query ID: " + res.queryId());
+
+ writer.writeLong(res.queryId());
+
+ Collection<?> items0 = res.items();
+
+ assert items0 != null;
+
+ writer.writeBoolean(res.last());
+
+ writer.writeInt(items0.size());
+
+ for (Object row0 : items0) {
+ if (row0 != null) {
+ Collection<?> row = (Collection<?>)row0;
+
+ writer.writeInt(row.size());
+
+ for (Object obj : row)
+ SqlListenerUtils.writeObject(writer, obj, true);
+ }
+ }
+ }
else if (res0 instanceof OdbcQueryCloseResult) {
OdbcQueryCloseResult res = (OdbcQueryCloseResult) res0;
@@ -320,4 +356,24 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
return writer.array();
}
+
+ /**
+ * @param writer Writer to use.
+ * @param affectedRows Affected rows.
+ */
+ private void writeAffectedRows(BinaryWriterExImpl writer, Collection<Long> affectedRows) {
+ if (ver.compareTo(OdbcConnectionContext.VER_2_3_0) < 0) {
+ long summ = 0;
+
+ for (Long value : affectedRows)
+ summ += value == null ? 0 : value;
+
+ writer.writeLong(summ);
+ }
+ else {
+ writer.writeInt(affectedRows.size());
+ for (Long value : affectedRows)
+ writer.writeLong(value);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
index c8f61dc..e86c7c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.odbc.odbc;
+import java.util.Collection;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.jetbrains.annotations.Nullable;
@@ -25,7 +26,7 @@ import org.jetbrains.annotations.Nullable;
*/
public class OdbcQueryExecuteBatchResult {
/** Rows affected. */
- private final long rowsAffected;
+ private final Collection<Long> affectedRows;
/** Index of the set which caused an error. */
private final long errorSetIdx;
@@ -37,23 +38,24 @@ public class OdbcQueryExecuteBatchResult {
private final String errorMessage;
/**
- * @param rowsAffected Number of rows affected by the query.
+ * @param affectedRows Number of rows affected by the query.
*/
- public OdbcQueryExecuteBatchResult(long rowsAffected) {
- this.rowsAffected = rowsAffected;
+ public OdbcQueryExecuteBatchResult(Collection<Long> affectedRows) {
+ this.affectedRows = affectedRows;
this.errorSetIdx = -1;
this.errorMessage = null;
this.errorCode = ClientListenerResponse.STATUS_SUCCESS;
}
/**
- * @param rowsAffected Number of rows affected by the query.
+ * @param affectedRows Number of rows affected by the query.
* @param errorSetIdx Sets processed.
* @param errorCode Error code.
* @param errorMessage Error message.
*/
- public OdbcQueryExecuteBatchResult(long rowsAffected, long errorSetIdx, int errorCode, String errorMessage) {
- this.rowsAffected = rowsAffected;
+ public OdbcQueryExecuteBatchResult(Collection<Long> affectedRows, long errorSetIdx, int errorCode,
+ String errorMessage) {
+ this.affectedRows = affectedRows;
this.errorSetIdx = errorSetIdx;
this.errorMessage = errorMessage;
this.errorCode = errorCode;
@@ -62,8 +64,8 @@ public class OdbcQueryExecuteBatchResult {
/**
* @return Number of rows affected by the query.
*/
- public long rowsAffected() {
- return rowsAffected;
+ public Collection<Long> affectedRows() {
+ return affectedRows;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteResult.java
index 38dd0b4..8182e97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteResult.java
@@ -29,15 +29,16 @@ public class OdbcQueryExecuteResult {
/** Fields metadata. */
private final Collection<OdbcColumnMeta> columnsMetadata;
- /** Rows affected by the query. */
- private final long affectedRows;
+ /** Rows affected by the statements. */
+ private final Collection<Long> affectedRows;
/**
* @param queryId Query ID.
* @param columnsMetadata Columns metadata.
* @param affectedRows Affected rows.
*/
- public OdbcQueryExecuteResult(long queryId, Collection<OdbcColumnMeta> columnsMetadata, long affectedRows) {
+ public OdbcQueryExecuteResult(long queryId, Collection<OdbcColumnMeta> columnsMetadata,
+ Collection<Long> affectedRows) {
this.queryId = queryId;
this.columnsMetadata = columnsMetadata;
this.affectedRows = affectedRows;
@@ -58,9 +59,9 @@ public class OdbcQueryExecuteResult {
}
/**
- * @return Number of rows affected by the query.
+ * @return Number of rows affected by the statements.
*/
- public long affectedRows() {
+ public Collection<Long> affectedRows() {
return affectedRows;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsRequest.java
new file mode 100644
index 0000000..5651f6d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsRequest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc.odbc;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * SQL listener query fetch request.
+ */
+public class OdbcQueryMoreResultsRequest extends OdbcRequest {
+ /** Query ID. */
+ private final long queryId;
+
+ /** Page size - maximum number of rows to return. */
+ private final int pageSize;
+
+ /**
+ * @param queryId Query ID.
+ * @param pageSize Page size.
+ */
+ public OdbcQueryMoreResultsRequest(long queryId, int pageSize) {
+ super(MORE_RESULTS);
+
+ this.queryId = queryId;
+ this.pageSize = pageSize;
+ }
+
+ /**
+ * @return Page size.
+ */
+ public int pageSize() {
+ return pageSize;
+ }
+
+ /**
+ * @return Query ID.
+ */
+ public long queryId() {
+ return queryId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(OdbcQueryMoreResultsRequest.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsResult.java
new file mode 100644
index 0000000..faa5e27
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryMoreResultsResult.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc.odbc;
+
+import java.util.Collection;
+
+/**
+ * SQL listener query fetch result.
+ */
+public class OdbcQueryMoreResultsResult {
+ /** Query ID. */
+ private final long queryId;
+
+ /** Query result rows. */
+ private final Collection<?> items;
+
+ /** Flag indicating the query has no non-fetched results. */
+ private final boolean last;
+
+ /**
+ * @param queryId Query ID.
+ * @param items Query result rows.
+ * @param last Flag indicating the query has no unfetched results.
+ */
+ public OdbcQueryMoreResultsResult(long queryId, Collection<?> items, boolean last){
+ this.queryId = queryId;
+ this.items = items;
+ this.last = last;
+ }
+
+ /**
+ * @return Query ID.
+ */
+ public long queryId() {
+ return queryId;
+ }
+
+ /**
+ * @return Query result rows.
+ */
+ public Collection<?> items() {
+ return items;
+ }
+
+ /**
+ * @return Flag indicating the query has no non-fetched results.
+ */
+ public boolean last() {
+ return last;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryResults.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryResults.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryResults.java
new file mode 100644
index 0000000..23788c7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryResults.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc.odbc;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
+
+/**
+ * ODBC result set
+ */
+public class OdbcQueryResults {
+ /** Current cursor. */
+ private final List<FieldsQueryCursor<List<?>>> cursors;
+
+ /** Rows affected. */
+ private final List<Long> rowsAffected;
+
+ /** Current result set. */
+ private OdbcResultSet currentResultSet;
+
+ /** Current result set index. */
+ private int currentResultSetIdx;
+
+ /**
+ * @param cursors Result set cursors.
+ */
+ OdbcQueryResults(List<FieldsQueryCursor<List<?>>> cursors) {
+ this.cursors = cursors;
+ this.currentResultSetIdx = 0;
+
+ rowsAffected = new ArrayList<>(cursors.size());
+
+ for (FieldsQueryCursor<List<?>> cursor : cursors)
+ rowsAffected.add(OdbcUtils.rowsAffected(cursor));
+
+ nextResultSet();
+ }
+
+ /**
+ * Get affected rows for all result sets.
+ * @return List of numbers of table rows affected by every statement.
+ */
+ public List<Long> rowsAffected() {
+ return rowsAffected;
+ }
+
+ /**
+ * @return {@code true} if any of the result sets still has non-fetched rows.
+ */
+ public boolean hasUnfetchedRows() {
+ if (currentResultSet != null && currentResultSet.hasUnfetchedRows())
+ return true;
+
+ for (FieldsQueryCursor<List<?>> cursor : cursors) {
+ QueryCursorImpl<List<?>> cursor0 = (QueryCursorImpl<List<?>>)cursor;
+
+ if (cursor0.isQuery())
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Close all cursors.
+ */
+ public void closeAll() {
+ for (FieldsQueryCursor<List<?>> cursor : cursors)
+ cursor.close();
+ }
+
+ /**
+ * @return Current result set.
+ */
+ public OdbcResultSet currentResultSet() {
+ return currentResultSet;
+ }
+
+ /**
+ * Move to next result set.
+ */
+ public void nextResultSet() {
+ currentResultSet = null;
+
+ if (currentResultSetIdx != cursors.size()) {
+ currentResultSet = new OdbcResultSet(cursors.get(currentResultSetIdx));
+ ++currentResultSetIdx;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequest.java
index 4b21b79..9b9aa01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequest.java
@@ -44,6 +44,9 @@ public class OdbcRequest extends ClientListenerRequestNoId {
/** Execute sql query with the batch of parameters. */
public static final int QRY_EXEC_BATCH = 8;
+ /** Get next result set. */
+ public static final int MORE_RESULTS = 9;
+
/** Command. */
private final int cmd;
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
index 32375fd..7f6b48d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
@@ -22,13 +22,13 @@ import java.sql.PreparedStatement;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.internal.GridKernalContext;
@@ -41,17 +41,16 @@ import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.apache.ignite.internal.processors.odbc.odbc.escape.OdbcEscapeUtils;
-import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
import org.apache.ignite.internal.processors.query.GridQueryIndexing;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.META_COLS;
import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.META_PARAMS;
import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.META_TBLS;
+import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.MORE_RESULTS;
import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.QRY_CLOSE;
import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.QRY_EXEC;
import static org.apache.ignite.internal.processors.odbc.odbc.OdbcRequest.QRY_EXEC_BATCH;
@@ -77,7 +76,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
private final int maxCursors;
/** Current queries cursors. */
- private final ConcurrentHashMap<Long, IgniteBiTuple<QueryCursor, Iterator>> qryCursors = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<Long, OdbcQueryResults> qryResults = new ConcurrentHashMap<>();
/** Distributed joins flag. */
private final boolean distributedJoins;
@@ -157,6 +156,9 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
case META_PARAMS:
return getParamsMeta((OdbcQueryGetParamsMetaRequest)req);
+
+ case MORE_RESULTS:
+ return moreResults((OdbcQueryMoreResultsRequest)req);
}
return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Unsupported ODBC request: " + req);
@@ -185,8 +187,8 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
{
try
{
- for (IgniteBiTuple<QueryCursor, Iterator> tuple : qryCursors.values())
- tuple.get1().close();
+ for (OdbcQueryResults res : qryResults.values())
+ res.closeAll();
}
finally {
busyLock.leaveBusy();
@@ -224,7 +226,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
* @return Response.
*/
private ClientListenerResponse executeQuery(OdbcQueryExecuteRequest req) {
- int cursorCnt = qryCursors.size();
+ int cursorCnt = qryResults.size();
if (maxCursors > 0 && cursorCnt >= maxCursors)
return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Too many open cursors (either close " +
@@ -243,26 +245,22 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
SqlFieldsQuery qry = makeQuery(req.schema(), sql, req.arguments());
- QueryCursorImpl<List<?>> qryCur = (QueryCursorImpl<List<?>>)ctx.query().querySqlFieldsNoCache(qry, true);
-
- long rowsAffected = 0;
+ List<FieldsQueryCursor<List<?>>> cursors = ctx.query().querySqlFieldsNoCache(qry, true, false);
- if (!qryCur.isQuery()) {
- rowsAffected = getRowsAffected(qryCur);
+ OdbcQueryResults results = new OdbcQueryResults(cursors);
- qryCur.close();
- }
+ if (!results.hasUnfetchedRows())
+ results.closeAll();
else
- qryCursors.put(qryId, new IgniteBiTuple<QueryCursor, Iterator>(qryCur, null));
+ qryResults.put(qryId, results);
- List<?> fieldsMeta = ((QueryCursorImpl) qryCur).fieldsMeta();
-
- OdbcQueryExecuteResult res = new OdbcQueryExecuteResult(qryId, convertMetadata(fieldsMeta), rowsAffected);
+ OdbcQueryExecuteResult res = new OdbcQueryExecuteResult(qryId, results.currentResultSet().fieldsMeta(),
+ results.rowsAffected());
return new OdbcResponse(res);
}
catch (Exception e) {
- qryCursors.remove(qryId);
+ qryResults.remove(qryId);
U.error(log, "Failed to execute SQL query [reqId=" + req.requestId() + ", req=" + req + ']', e);
@@ -277,7 +275,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
* @return Response.
*/
private ClientListenerResponse executeBatchQuery(OdbcQueryExecuteBatchRequest req) {
- long rowsAffected = 0;
+ List<Long> rowsAffected = new ArrayList<>(req.arguments().length);
int currentSet = 0;
try {
@@ -304,10 +302,10 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
throw new IgniteException("Batching of parameters only supported for DML statements. [query=" +
req.sqlQuery() + ']');
- rowsAffected += getRowsAffected(qryCur);
+ rowsAffected.add(OdbcUtils.rowsAffected(qryCur));
for (currentSet = 1; currentSet < paramSet.length; ++currentSet)
- rowsAffected += executeQuery(qry, paramSet[currentSet]);
+ rowsAffected.add(executeQuery(qry, paramSet[currentSet]));
OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected);
@@ -331,29 +329,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
QueryCursor<List<?>> cur = ctx.query().querySqlFieldsNoCache(qry, true);
- return getRowsAffected(cur);
- }
-
- /**
- * Get affected rows for DML statement.
- * @param qryCur Cursor.
- * @return Number of table rows affected.
- */
- private static long getRowsAffected(QueryCursor<List<?>> qryCur) {
- Iterator<List<?>> iter = qryCur.iterator();
-
- if (iter.hasNext()) {
- List<?> res = iter.next();
-
- if (res.size() > 0) {
- Long affected = (Long) res.get(0);
-
- if (affected != null)
- return affected;
- }
- }
-
- return 0;
+ return OdbcUtils.rowsAffected(cur);
}
/**
@@ -366,20 +342,20 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
long queryId = req.queryId();
try {
- IgniteBiTuple<QueryCursor, Iterator> tuple = qryCursors.get(queryId);
+ OdbcQueryResults results = qryResults.get(queryId);
- if (tuple == null)
+ if (results == null)
return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN,
"Failed to find query with ID: " + queryId);
- CloseCursor(tuple, queryId);
+ CloseCursor(results, queryId);
OdbcQueryCloseResult res = new OdbcQueryCloseResult(queryId);
return new OdbcResponse(res);
}
catch (Exception e) {
- qryCursors.remove(queryId);
+ qryResults.remove(queryId);
U.error(log, "Failed to close SQL query [reqId=" + req.requestId() + ", req=" + queryId + ']', e);
@@ -396,34 +372,21 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
private ClientListenerResponse fetchQuery(OdbcQueryFetchRequest req) {
try {
long queryId = req.queryId();
- IgniteBiTuple<QueryCursor, Iterator> tuple = qryCursors.get(queryId);
+ OdbcQueryResults results = qryResults.get(queryId);
- if (tuple == null)
+ if (results == null)
return new OdbcResponse(ClientListenerResponse.STATUS_FAILED,
"Failed to find query with ID: " + queryId);
- Iterator iter = tuple.get2();
+ OdbcResultSet set = results.currentResultSet();
- if (iter == null) {
- QueryCursor cur = tuple.get1();
+ List<Object> items = set.fetch(req.pageSize());
- assert(cur != null);
-
- iter = cur.iterator();
-
- tuple.put(cur, iter);
- }
-
- List<Object> items = new ArrayList<>();
-
- for (int i = 0; i < req.pageSize() && iter.hasNext(); ++i)
- items.add(iter.next());
-
- boolean lastPage = !iter.hasNext();
+ boolean lastPage = !set.hasUnfetchedRows();
// Automatically closing cursor if no more data is available.
- if (lastPage)
- CloseCursor(tuple, queryId);
+ if (!results.hasUnfetchedRows())
+ CloseCursor(results, queryId);
OdbcQueryFetchResult res = new OdbcQueryFetchResult(queryId, items, lastPage);
@@ -581,18 +544,55 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
}
/**
+ * {@link OdbcQueryMoreResultsRequest} command handler.
+ *
+ * @param req Execute query request.
+ * @return Response.
+ */
+ private ClientListenerResponse moreResults(OdbcQueryMoreResultsRequest req) {
+ try {
+ long queryId = req.queryId();
+ OdbcQueryResults results = qryResults.get(queryId);
+
+ if (results == null)
+ return new OdbcResponse(ClientListenerResponse.STATUS_FAILED,
+ "Failed to find query with ID: " + queryId);
+
+ results.nextResultSet();
+
+ OdbcResultSet set = results.currentResultSet();
+
+ List<Object> items = set.fetch(req.pageSize());
+
+ boolean lastPage = !set.hasUnfetchedRows();
+
+ // Automatically closing cursor if no more data is available.
+ if (!results.hasUnfetchedRows())
+ CloseCursor(results, queryId);
+
+ OdbcQueryMoreResultsResult res = new OdbcQueryMoreResultsResult(queryId, items, lastPage);
+
+ return new OdbcResponse(res);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to get more SQL query results [reqId=" +
+ req.requestId() + ", req=" + req + ']', e);
+
+ return exceptionToResult(e);
+ }
+ }
+
+ /**
* Close cursor.
- * @param tuple Query map element.
+ * @param results Query map element.
* @param queryId Query ID.
*/
- private void CloseCursor(IgniteBiTuple<QueryCursor, Iterator> tuple, long queryId) {
- QueryCursor cur = tuple.get1();
-
- assert(cur != null);
+ private void CloseCursor(OdbcQueryResults results, long queryId) {
+ assert(results != null);
- cur.close();
+ results.closeAll();
- qryCursors.remove(queryId);
+ qryResults.remove(queryId);
}
/**
@@ -655,27 +655,6 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
}
/**
- * Convert metadata in collection from {@link GridQueryFieldMetadata} to
- * {@link OdbcColumnMeta}.
- *
- * @param meta Internal query field metadata.
- * @return Odbc query field metadata.
- */
- private static Collection<OdbcColumnMeta> convertMetadata(Collection<?> meta) {
- List<OdbcColumnMeta> res = new ArrayList<>();
-
- if (meta != null) {
- for (Object info : meta) {
- assert info instanceof GridQueryFieldMetadata;
-
- res.add(new OdbcColumnMeta((GridQueryFieldMetadata)info));
- }
- }
-
- return res;
- }
-
- /**
* Checks whether string matches SQL pattern.
*
* @param str String.
@@ -694,7 +673,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
* @param e Exception to convert.
* @return resulting {@link OdbcResponse}.
*/
- private OdbcResponse exceptionToBatchResult(Exception e, long rowsAffected, long currentSet) {
+ private OdbcResponse exceptionToBatchResult(Exception e, Collection<Long> rowsAffected, long currentSet) {
OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
OdbcUtils.tryRetrieveSqlErrorCode(e), OdbcUtils.tryRetrieveH2ErrorMessage(e));
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcResultSet.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcResultSet.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcResultSet.java
new file mode 100644
index 0000000..66b0776
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcResultSet.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.odbc.odbc;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
+import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
+
+/**
+ * Represents single result set.
+ */
+public class OdbcResultSet {
+ /** Cursor. */
+ private final QueryCursorImpl<List<?>> cursor;
+
+ /** Current iterator. */
+ private Iterator iter;
+
+ /**
+ * Constructor.
+ * @param cursor Result set cursor.
+ */
+ OdbcResultSet(FieldsQueryCursor<List<?>> cursor) {
+ assert cursor instanceof QueryCursorImpl;
+
+ this.cursor = (QueryCursorImpl<List<?>>)cursor;
+
+ if (this.cursor.isQuery())
+ iter = this.cursor.iterator();
+ else
+ iter = null;
+ }
+
+ /**
+ * @return {@code true} if has non-fetched rows.
+ */
+ public boolean hasUnfetchedRows() {
+ return iter != null && iter.hasNext();
+ }
+
+ /**
+ * @return Fields metadata of the current result set.
+ */
+ public Collection<OdbcColumnMeta> fieldsMeta() {
+ return convertMetadata(cursor.fieldsMeta());
+ }
+
+ /**
+ * Fetch up to specified number of rows of result set.
+ * @param maxSize Maximum number of records to fetch.
+ * @return List of fetched records.
+ */
+ public List<Object> fetch(int maxSize) {
+ List<Object> items = new ArrayList<>(maxSize);
+
+ if (iter == null)
+ return items;
+
+ for (int i = 0; i < maxSize && iter.hasNext(); ++i)
+ items.add(iter.next());
+
+ return items;
+ }
+
+ /**
+ * Convert metadata in collection from {@link GridQueryFieldMetadata} to
+ * {@link OdbcColumnMeta}.
+ *
+ * @param meta Internal query field metadata.
+ * @return Odbc query field metadata.
+ */
+ private static Collection<OdbcColumnMeta> convertMetadata(Collection<GridQueryFieldMetadata> meta) {
+ List<OdbcColumnMeta> res = new ArrayList<>();
+
+ if (meta != null) {
+ for (GridQueryFieldMetadata info : meta)
+ res.add(new OdbcColumnMeta(info));
+ }
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
index 98fa045..4aa864d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
@@ -17,7 +17,11 @@
package org.apache.ignite.internal.processors.odbc.odbc;
+import java.util.Iterator;
+import java.util.List;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.odbc.SqlListenerDataTypes;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
@@ -181,4 +185,31 @@ public class OdbcUtils {
return msg;
}
+
+ /**
+ * Get affected rows for statement.
+ * @param qryCur Cursor.
+ * @return Number of table rows affected, if the query is DML, and -1 otherwise.
+ */
+ public static long rowsAffected(QueryCursor<List<?>> qryCur) {
+ QueryCursorImpl<List<?>> qryCur0 = (QueryCursorImpl<List<?>>)qryCur;
+
+ if (qryCur0.isQuery())
+ return -1;
+
+ Iterator<List<?>> iter = qryCur0.iterator();
+
+ if (iter.hasNext()) {
+ List<?> res = iter.next();
+
+ if (res.size() > 0) {
+ Long affected = (Long) res.get(0);
+
+ if (affected != null)
+ return affected;
+ }
+ }
+
+ return 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/examples/odbc-example/src/odbc_example.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/examples/odbc-example/src/odbc_example.cpp b/modules/platforms/cpp/examples/odbc-example/src/odbc_example.cpp
index 29c2a4a..76b5c18 100644
--- a/modules/platforms/cpp/examples/odbc-example/src/odbc_example.cpp
+++ b/modules/platforms/cpp/examples/odbc-example/src/odbc_example.cpp
@@ -169,7 +169,7 @@ void GetDataWithOdbc(SQLHDBC dbc, const std::string& query)
/**
* Populate Person cache with sample data.
- *
+ *
* @param dbc Database connection.
*/
void PopulatePerson(SQLHDBC dbc)
@@ -387,7 +387,7 @@ void PopulatePerson(SQLHDBC dbc)
/**
* Populate Organization cache with sample data.
- *
+ *
* @param dbc Database connection.
*/
void PopulateOrganization(SQLHDBC dbc)
@@ -525,7 +525,7 @@ void DeletePerson(SQLHDBC dbc, int64_t key)
/**
* Query tables.
- *
+ *
* @param dbc Database connection.
*/
void QueryData(SQLHDBC dbc)
@@ -552,7 +552,7 @@ void QueryData(SQLHDBC dbc)
*
* @return Exit code.
*/
-int main()
+int main()
{
IgniteConfiguration cfg;
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc-test/src/queries_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index 707669d..c6097e0 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -557,19 +557,27 @@ struct QueriesTestSuiteFixture
if (!SQL_SUCCEEDED(ret))
BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
- SQLLEN affected = 0;
- ret = SQLRowCount(stmt, &affected);
+ SQLLEN totallyAffected = 0;
- if (!SQL_SUCCEEDED(ret))
- BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+ do
+ {
+ SQLLEN affected = 0;
+ ret = SQLRowCount(stmt, &affected);
- BOOST_CHECK_EQUAL(affected, expectedToAffect);
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
- BOOST_CHECKPOINT("Getting next result set");
- ret = SQLMoreResults(stmt);
+ totallyAffected += affected;
- if (ret != SQL_NO_DATA)
- BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+ BOOST_CHECKPOINT("Getting next result set");
+
+ ret = SQLMoreResults(stmt);
+
+ if (ret != SQL_SUCCESS && ret != SQL_NO_DATA)
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+ } while (ret != SQL_NO_DATA);
+
+ BOOST_CHECK_EQUAL(totallyAffected, expectedToAffect);
BOOST_CHECKPOINT("Resetting parameters.");
ret = SQLFreeStmt(stmt, SQL_RESET_PARAMS);
@@ -2198,4 +2206,178 @@ BOOST_AUTO_TEST_CASE(TestAffectedRows)
BOOST_CHECK_EQUAL(affected, 0);
}
+BOOST_AUTO_TEST_CASE(TestMultipleSelects)
+{
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+ const int stmtCnt = 10;
+
+ std::stringstream stream;
+ for (int i = 0; i < stmtCnt; ++i)
+ stream << "select " << i << "; ";
+
+ stream << '\0';
+
+ std::string query0 = stream.str();
+ std::vector<SQLCHAR> query(query0.begin(), query0.end());
+
+ SQLRETURN ret = SQLExecDirect(stmt, &query[0], SQL_NTS);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ long res = 0;
+
+ BOOST_CHECKPOINT("Binding column");
+ ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &res, 0, 0);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ for (long i = 0; i < stmtCnt; ++i)
+ {
+ ret = SQLFetch(stmt);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ BOOST_CHECK_EQUAL(res, i);
+
+ ret = SQLFetch(stmt);
+
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+
+ ret = SQLMoreResults(stmt);
+
+ if (i < stmtCnt - 1 && !SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+ else if (i == stmtCnt - 1)
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+ }
+}
+
+BOOST_AUTO_TEST_CASE(TestMultipleMixedStatements)
+{
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+ const int stmtCnt = 10;
+
+ std::stringstream stream;
+ for (int i = 0; i < stmtCnt; ++i)
+ stream << "select " << i << "; insert into TestType(_key) values(" << i << "); ";
+
+ stream << '\0';
+
+ std::string query0 = stream.str();
+ std::vector<SQLCHAR> query(query0.begin(), query0.end());
+
+ SQLRETURN ret = SQLExecDirect(stmt, &query[0], SQL_NTS);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ long res = 0;
+
+ BOOST_CHECKPOINT("Binding column");
+ ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &res, 0, 0);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ for (long i = 0; i < stmtCnt; ++i)
+ {
+ ret = SQLFetch(stmt);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ BOOST_CHECK_EQUAL(res, i);
+
+ ret = SQLFetch(stmt);
+
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+
+ ret = SQLMoreResults(stmt);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ SQLLEN affected = 0;
+ ret = SQLRowCount(stmt, &affected);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ BOOST_CHECK_EQUAL(affected, 1);
+
+ ret = SQLFetch(stmt);
+
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+
+ ret = SQLMoreResults(stmt);
+
+ if (i < stmtCnt - 1 && !SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+ else if (i == stmtCnt - 1)
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+ }
+}
+
+BOOST_AUTO_TEST_CASE(TestMultipleMixedStatementsNoFetch)
+{
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+ const int stmtCnt = 10;
+
+ std::stringstream stream;
+ for (int i = 0; i < stmtCnt; ++i)
+ stream << "select " << i << "; insert into TestType(_key) values(" << i << "); ";
+
+ stream << '\0';
+
+ std::string query0 = stream.str();
+ std::vector<SQLCHAR> query(query0.begin(), query0.end());
+
+ SQLRETURN ret = SQLExecDirect(stmt, &query[0], SQL_NTS);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ long res = 0;
+
+ BOOST_CHECKPOINT("Binding column");
+ ret = SQLBindCol(stmt, 1, SQL_C_SLONG, &res, 0, 0);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ for (long i = 0; i < stmtCnt; ++i)
+ {
+ ret = SQLMoreResults(stmt);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ SQLLEN affected = 0;
+ ret = SQLRowCount(stmt, &affected);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ BOOST_CHECK_EQUAL(affected, 1);
+
+ ret = SQLFetch(stmt);
+
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+
+ ret = SQLMoreResults(stmt);
+
+ if (i < stmtCnt - 1 && !SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+ else if (i == stmtCnt - 1)
+ BOOST_CHECK_EQUAL(ret, SQL_NO_DATA);
+ }
+}
+
+
BOOST_AUTO_TEST_SUITE_END()
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
index dda0ba9..8d6c906 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
@@ -60,7 +60,9 @@ namespace ignite
GET_PARAMS_METADATA = 7,
- EXECUTE_SQL_QUERY_BATCH = 8
+ EXECUTE_SQL_QUERY_BATCH = 8,
+
+ QUERY_MORE_RESULTS = 9
};
};
@@ -387,6 +389,47 @@ namespace ignite
};
/**
+ * Query fetch request.
+ */
+ class QueryMoreResultsRequest
+ {
+ public:
+ /**
+ * Constructor.
+ *
+ * @param queryId Query ID.
+ * @param pageSize Required page size.
+ */
+ QueryMoreResultsRequest(int64_t queryId, int32_t pageSize) :
+ queryId(queryId),
+ pageSize(pageSize)
+ {
+ // No-op.
+ }
+
+ /**
+ * Destructor.
+ */
+ ~QueryMoreResultsRequest()
+ {
+ // No-op.
+ }
+
+ /**
+ * Write request using provided writer.
+ * @param writer Writer.
+ */
+ void Write(impl::binary::BinaryWriterImpl& writer) const;
+
+ private:
+ /** Query ID. */
+ int64_t queryId;
+
+ /** SQL query. */
+ int32_t pageSize;
+ };
+
+ /**
* General response.
*/
class Response
@@ -575,7 +618,7 @@ namespace ignite
* Get affected rows number.
* @return Number of rows affected by the query.
*/
- int64_t GetAffectedRows()
+ const std::vector<int64_t>& GetAffectedRows()
{
return affectedRows;
}
@@ -585,7 +628,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver);
/** Query ID. */
int64_t queryId;
@@ -594,7 +637,7 @@ namespace ignite
meta::ColumnMetaVector meta;
/** Number of affected rows. */
- int64_t affectedRows;
+ std::vector<int64_t> affectedRows;
};
/**
@@ -617,7 +660,7 @@ namespace ignite
* Affected rows.
* @return Affected rows.
*/
- int64_t GetAffectedRows() const
+ const std::vector<int64_t>& GetAffectedRows() const
{
return affectedRows;
}
@@ -628,7 +671,7 @@ namespace ignite
*/
int64_t GetErrorSetIdx() const
{
- return affectedRows;
+ return static_cast<int64_t>(affectedRows.size());
}
/**
@@ -658,7 +701,7 @@ namespace ignite
virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver);
/** Affected rows. */
- int64_t affectedRows;
+ std::vector<int64_t> affectedRows;
/** Index of the set which caused an error. */
int64_t errorSetIdx;
@@ -817,6 +860,46 @@ namespace ignite
/** Columns metadata. */
std::vector<int8_t> typeIds;
};
+
+ /**
+ * Query fetch response.
+ */
+ class QueryMoreResultsResponse : public Response
+ {
+ public:
+ /**
+ * Constructor.
+ * @param resultPage Result page.
+ */
+ QueryMoreResultsResponse(ResultPage& resultPage);
+
+ /**
+ * Destructor.
+ */
+ virtual ~QueryMoreResultsResponse();
+
+ /**
+ * Get query ID.
+ * @return Query ID.
+ */
+ int64_t GetQueryId() const
+ {
+ return queryId;
+ }
+
+ private:
+ /**
+ * Read response using provided reader.
+ * @param reader Reader.
+ */
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
+
+ /** Query ID. */
+ int64_t queryId;
+
+ /** Result page. */
+ ResultPage& resultPage;
+ };
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
index 5e741ed..1e6c869 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/batch_query.h
@@ -106,6 +106,13 @@ namespace ignite
virtual int64_t AffectedRows() const;
/**
+ * Move to the next result set.
+ *
+ * @return Operaion result.
+ */
+ virtual SqlResult::Type NextResultSet();
+
+ /**
* Get SQL query string.
*
* @return SQL query string.
@@ -142,16 +149,13 @@ namespace ignite
meta::ColumnMetaVector resultMeta;
/** Number of rows affected. */
- int64_t rowsAffected;
+ std::vector<int64_t> rowsAffected;
- /** Number of parameter sets successfully processed. */
- int64_t setsProcessed;
+ /** Rows affected index. */
+ size_t rowsAffectedIdx;
/** Query executed. */
bool executed;
-
- /** Data retrieved. */
- bool dataRetrieved;
};
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/column_metadata_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/column_metadata_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/column_metadata_query.h
index 875b1ce..d742490 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/column_metadata_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/column_metadata_query.h
@@ -105,6 +105,13 @@ namespace ignite
*/
virtual int64_t AffectedRows() const;
+ /**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet();
+
private:
IGNITE_NO_COPY_ASSIGNMENT(ColumnMetadataQuery);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
index 5a4a978..c476000 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/data_query.h
@@ -106,6 +106,13 @@ namespace ignite
virtual int64_t AffectedRows() const;
/**
+ * Move to the next result set.
+ *
+ * @return Operaion result.
+ */
+ virtual SqlResult::Type NextResultSet();
+
+ /**
* Get SQL query string.
*
* @return SQL query string.
@@ -139,6 +146,13 @@ namespace ignite
* @return Result.
*/
SqlResult::Type MakeRequestFetch();
+
+ /**
+ * Make next result set request and use response to set internal state.
+ *
+ * @return Result.
+ */
+ SqlResult::Type MakeRequestMoreResults();
/**
* Close query.
@@ -163,7 +177,13 @@ namespace ignite
std::auto_ptr<Cursor> cursor;
/** Number of rows affected. */
- int64_t rowsAffected;
+ std::vector<int64_t> rowsAffected;
+
+ /** Rows affected index. */
+ size_t rowsAffectedIdx;
+
+ /** Cached next result page. */
+ std::auto_ptr<ResultPage> cachedNextPage;
};
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/foreign_keys_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/foreign_keys_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/foreign_keys_query.h
index 7d60728..9abd8b2 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/foreign_keys_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/foreign_keys_query.h
@@ -105,6 +105,13 @@ namespace ignite
* @return Number of rows affected by the statement.
*/
virtual int64_t AffectedRows() const;
+
+ /**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet();
private:
IGNITE_NO_COPY_ASSIGNMENT(ForeignKeysQuery);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/primary_keys_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/primary_keys_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/primary_keys_query.h
index 65bac33..42f7e26 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/primary_keys_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/primary_keys_query.h
@@ -102,6 +102,13 @@ namespace ignite
* @return Number of rows affected by the statement.
*/
virtual int64_t AffectedRows() const;
+
+ /**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet();
private:
IGNITE_NO_COPY_ASSIGNMENT(PrimaryKeysQuery);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
index 701f5c8..9d54b90 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/query.h
@@ -131,6 +131,13 @@ namespace ignite
virtual int64_t AffectedRows() const = 0;
/**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet() = 0;
+
+ /**
* Get query type.
*
* @return Query type.
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/special_columns_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/special_columns_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/special_columns_query.h
index 0f6660f..d6a5c44 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/special_columns_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/special_columns_query.h
@@ -105,6 +105,13 @@ namespace ignite
*/
virtual int64_t AffectedRows() const;
+ /**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet();
+
private:
IGNITE_NO_COPY_ASSIGNMENT(SpecialColumnsQuery);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/table_metadata_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/table_metadata_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/table_metadata_query.h
index acd3f49..759bfd6 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/table_metadata_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/table_metadata_query.h
@@ -106,6 +106,13 @@ namespace ignite
*/
virtual int64_t AffectedRows() const;
+ /**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet();
+
private:
IGNITE_NO_COPY_ASSIGNMENT(TableMetadataQuery);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
index 00cca08..974ee01 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/query/type_info_query.h
@@ -96,6 +96,13 @@ namespace ignite
*/
virtual int64_t AffectedRows() const;
+ /**
+ * Move to the next result set.
+ *
+ * @return Operatoin result.
+ */
+ virtual SqlResult::Type NextResultSet();
+
private:
IGNITE_NO_COPY_ASSIGNMENT(TypeInfoQuery);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
index 27e883d..6d4b3ab 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/statement.h
@@ -264,11 +264,11 @@ namespace ignite
bool DataAvailable() const;
/**
- * Next results.
+ * More results.
*
* Move to next result set or affected rows number.
*/
- void NextResults();
+ void MoreResults();
/**
* Get column attribute.
@@ -581,7 +581,7 @@ namespace ignite
*
* @return Operation result.
*/
- SqlResult::Type InternalNextResults();
+ SqlResult::Type InternalMoreResults();
/**
* Get column attribute.
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/cursor.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/cursor.cpp b/modules/platforms/cpp/odbc/src/cursor.cpp
index b41f5b1..09e96cb 100644
--- a/modules/platforms/cpp/odbc/src/cursor.cpp
+++ b/modules/platforms/cpp/odbc/src/cursor.cpp
@@ -66,7 +66,7 @@ namespace ignite
bool Cursor::IsClosedRemotely() const
{
- return currentPage.get() && currentPage->IsLast();
+ return !currentPage.get() || currentPage->IsLast();
}
void Cursor::UpdateData(std::auto_ptr<ResultPage>& newPage)
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/message.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp
index 4767c74..5595ddb 100644
--- a/modules/platforms/cpp/odbc/src/message.cpp
+++ b/modules/platforms/cpp/odbc/src/message.cpp
@@ -18,6 +18,29 @@
#include "ignite/odbc/message.h"
#include "ignite/odbc/utility.h"
+namespace
+{
+ using namespace ignite;
+ using namespace odbc;
+
+ void ReadAffectedRows(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& protocolVersion,
+ std::vector<int64_t>& affectedRows)
+ {
+ affectedRows.clear();
+
+ if (protocolVersion < ProtocolVersion::VERSION_2_3_0)
+ affectedRows.push_back(reader.ReadInt64());
+ else
+ {
+ int32_t len = reader.ReadInt32();
+
+ affectedRows.reserve(static_cast<size_t>(len));
+ for (int32_t i = 0; i < len; ++i)
+ affectedRows.push_back(reader.ReadInt64());
+ }
+ }
+}
+
namespace ignite
{
namespace odbc
@@ -152,6 +175,7 @@ namespace ignite
void QueryFetchRequest::Write(impl::binary::BinaryWriterImpl& writer) const
{
writer.WriteInt8(RequestType::FETCH_SQL_QUERY);
+
writer.WriteInt64(queryId);
writer.WriteInt32(pageSize);
}
@@ -212,6 +236,14 @@ namespace ignite
writer.WriteObject<std::string>(sqlQuery);
}
+ void QueryMoreResultsRequest::Write(impl::binary::BinaryWriterImpl& writer) const
+ {
+ writer.WriteInt8(RequestType::QUERY_MORE_RESULTS);
+
+ writer.WriteInt64(queryId);
+ writer.WriteInt32(pageSize);
+ }
+
Response::Response() :
status(ResponseStatus::UNKNOWN_ERROR),
error()
@@ -299,13 +331,13 @@ namespace ignite
// No-op.
}
- void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
+ void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver)
{
queryId = reader.ReadInt64();
meta::ReadColumnMetaVector(reader, meta);
- affectedRows = reader.ReadInt64();
+ ReadAffectedRows(reader, ver, affectedRows);
}
QueryExecuteBatchResponse::QueryExecuteBatchResponse():
@@ -325,7 +357,8 @@ namespace ignite
void QueryExecuteBatchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver)
{
bool success = reader.ReadBool();
- affectedRows = reader.ReadInt64();
+
+ ReadAffectedRows(reader, ver, affectedRows);
if (!success)
{
@@ -337,7 +370,9 @@ namespace ignite
}
}
- QueryFetchResponse::QueryFetchResponse(ResultPage& resultPage): queryId(0), resultPage(resultPage)
+ QueryFetchResponse::QueryFetchResponse(ResultPage& resultPage) :
+ queryId(0),
+ resultPage(resultPage)
{
// No-op.
}
@@ -398,6 +433,25 @@ namespace ignite
{
utility::ReadByteArray(reader, typeIds);
}
+
+ QueryMoreResultsResponse::QueryMoreResultsResponse(ResultPage & resultPage) :
+ queryId(0),
+ resultPage(resultPage)
+ {
+ // No-op.
+ }
+
+ QueryMoreResultsResponse::~QueryMoreResultsResponse()
+ {
+ // No-op.
+ }
+
+ void QueryMoreResultsResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
+ {
+ queryId = reader.ReadInt64();
+
+ resultPage.Read(reader);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/odbc.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/odbc.cpp b/modules/platforms/cpp/odbc/src/odbc.cpp
index 8121a3b..1480d0b 100644
--- a/modules/platforms/cpp/odbc/src/odbc.cpp
+++ b/modules/platforms/cpp/odbc/src/odbc.cpp
@@ -593,7 +593,7 @@ namespace ignite
if (!statement)
return SQL_INVALID_HANDLE;
- statement->NextResults();
+ statement->MoreResults();
return statement->GetDiagnosticRecords().GetReturnCode();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/batch_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/batch_query.cpp b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
index fc8fda4..29d11ca 100644
--- a/modules/platforms/cpp/odbc/src/query/batch_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
@@ -34,10 +34,9 @@ namespace ignite
sql(sql),
params(params),
resultMeta(),
- rowsAffected(0),
- setsProcessed(0),
- executed(false),
- dataRetrieved(false)
+ rowsAffected(),
+ rowsAffectedIdx(0),
+ executed(false)
{
// No-op.
}
@@ -62,6 +61,9 @@ namespace ignite
int32_t processed = 0;
+ rowsAffected.clear();
+ rowsAffected.reserve(static_cast<size_t>(params.GetParamSetSize()));
+
do {
int32_t currentPageSize = std::min(maxPageSize, rowNum - processed);
bool lastPage = currentPageSize == rowNum - processed;
@@ -71,7 +73,7 @@ namespace ignite
processed += currentPageSize;
} while (res == SqlResult::AI_SUCCESS && processed < rowNum);
- params.SetParamsProcessed(static_cast<SqlUlen>(setsProcessed));
+ params.SetParamsProcessed(static_cast<SqlUlen>(rowsAffected.size()));
return res;
}
@@ -90,17 +92,7 @@ namespace ignite
return SqlResult::AI_ERROR;
}
- if (dataRetrieved)
- return SqlResult::AI_NO_DATA;
-
- app::ColumnBindingMap::iterator it = columnBindings.find(1);
-
- if (it != columnBindings.end())
- it->second.PutInt64(rowsAffected);
-
- dataRetrieved = true;
-
- return SqlResult::AI_SUCCESS;
+ return SqlResult::AI_NO_DATA;
}
SqlResult::Type BatchQuery::GetColumn(uint16_t columnIdx, app::ApplicationDataBuffer& buffer)
@@ -112,31 +104,18 @@ namespace ignite
return SqlResult::AI_ERROR;
}
- if (dataRetrieved)
- {
- diag.AddStatusRecord(SqlState::S24000_INVALID_CURSOR_STATE,
- "Cursor has reached end of the result set.");
-
- return SqlResult::AI_ERROR;
- }
-
- if (columnIdx != 1)
- {
- std::stringstream builder;
- builder << "Column with id " << columnIdx << " is not available in result set.";
-
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, builder.str());
+ diag.AddStatusRecord(SqlState::S24000_INVALID_CURSOR_STATE,
+ "Cursor has reached end of the result set.");
- return SqlResult::AI_ERROR;
- }
-
- buffer.PutInt64(rowsAffected);
-
- return SqlResult::AI_SUCCESS;
+ return SqlResult::AI_ERROR;
}
SqlResult::Type BatchQuery::Close()
{
+ executed = false;
+ rowsAffected.clear();
+ rowsAffectedIdx = 0;
+
return SqlResult::AI_SUCCESS;
}
@@ -147,7 +126,21 @@ namespace ignite
int64_t BatchQuery::AffectedRows() const
{
- return rowsAffected;
+ int64_t affected = rowsAffectedIdx < rowsAffected.size() ? rowsAffected[rowsAffectedIdx] : 0;
+ return affected < 0 ? 0 : affected;
+ }
+
+ SqlResult::Type BatchQuery::NextResultSet()
+ {
+ if (rowsAffectedIdx + 1 >= rowsAffected.size())
+ {
+ Close();
+ return SqlResult::AI_NO_DATA;
+ }
+
+ ++rowsAffectedIdx;
+
+ return SqlResult::AI_SUCCESS;
}
SqlResult::Type BatchQuery::MakeRequestExecuteBatch(SqlUlen begin, SqlUlen end, bool last)
@@ -183,25 +176,20 @@ namespace ignite
return SqlResult::AI_ERROR;
}
- rowsAffected += rsp.GetAffectedRows();
- LOG_MSG("rowsAffected: " << rowsAffected);
+ rowsAffected.insert(rowsAffected.end(), rsp.GetAffectedRows().begin(), rsp.GetAffectedRows().end());
+ LOG_MSG("Affected rows list size: " << rowsAffected.size());
if (!rsp.GetErrorMessage().empty())
{
LOG_MSG("Error: " << rsp.GetErrorMessage());
-
- setsProcessed += rsp.GetErrorSetIdx();
- LOG_MSG("setsProcessed: " << setsProcessed);
+ LOG_MSG("Sets Processed: " << rowsAffected.size());
diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetErrorCode()), rsp.GetErrorMessage(),
- static_cast<int32_t>(setsProcessed), 0);
+ static_cast<int32_t>(rowsAffected.size()), 0);
return SqlResult::AI_SUCCESS_WITH_INFO;
}
- setsProcessed += end - begin;
- LOG_MSG("setsProcessed: " << setsProcessed);
-
return SqlResult::AI_SUCCESS;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp b/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
index 0910612..0a09159 100644
--- a/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
@@ -289,6 +289,11 @@ namespace ignite
return 0;
}
+ SqlResult::Type ColumnMetadataQuery::NextResultSet()
+ {
+ return SqlResult::AI_NO_DATA;
+ }
+
SqlResult::Type ColumnMetadataQuery::MakeRequestGetColumnsMeta()
{
QueryGetColumnsMetaRequest req(schema, table, column);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/data_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/data_query.cpp b/modules/platforms/cpp/odbc/src/query/data_query.cpp
index 80fcc69..012b026 100644
--- a/modules/platforms/cpp/odbc/src/query/data_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/data_query.cpp
@@ -36,7 +36,9 @@ namespace ignite
params(params),
resultMeta(),
cursor(),
- rowsAffected(0)
+ rowsAffected(),
+ rowsAffectedIdx(0),
+ cachedNextPage()
{
// No-op.
}
@@ -79,10 +81,15 @@ namespace ignite
if (cursor->NeedDataUpdate())
{
- SqlResult::Type result = MakeRequestFetch();
+ if (cachedNextPage.get())
+ cursor->UpdateData(cachedNextPage);
+ else
+ {
+ SqlResult::Type result = MakeRequestFetch();
- if (result != SqlResult::AI_SUCCESS)
- return result;
+ if (result != SqlResult::AI_SUCCESS)
+ return result;
+ }
}
if (!cursor->HasData())
@@ -168,6 +175,10 @@ namespace ignite
cursor.reset();
resultMeta.clear();
+
+ rowsAffectedIdx = 0;
+
+ rowsAffected.clear();
}
return result;
@@ -180,7 +191,25 @@ namespace ignite
int64_t DataQuery::AffectedRows() const
{
- return rowsAffected;
+ int64_t affected = rowsAffectedIdx < rowsAffected.size() ? rowsAffected[rowsAffectedIdx] : 0;
+ return affected < 0 ? 0 : affected;
+ }
+
+ SqlResult::Type DataQuery::NextResultSet()
+ {
+ if (rowsAffectedIdx + 1 >= rowsAffected.size())
+ {
+ InternalClose();
+
+ return SqlResult::AI_NO_DATA;
+ }
+
+ SqlResult::Type res = MakeRequestMoreResults();
+
+ if (res == SqlResult::AI_SUCCESS)
+ ++rowsAffectedIdx;
+
+ return res;
}
SqlResult::Type DataQuery::MakeRequestExecute()
@@ -221,7 +250,7 @@ namespace ignite
rowsAffected = rsp.GetAffectedRows();
LOG_MSG("Query id: " << rsp.GetQueryId());
- LOG_MSG("Affected Rows: " << rowsAffected);
+ LOG_MSG("Affected Rows list size: " << rowsAffected.size());
for (size_t i = 0; i < resultMeta.size(); ++i)
{
@@ -231,10 +260,9 @@ namespace ignite
<< "\n[" << i << "] ColumnType: " << static_cast<int32_t>(resultMeta[i].GetDataType()));
}
- if (rowsAffected > 0)
- cursor.reset();
- else
- cursor.reset(new Cursor(rsp.GetQueryId()));
+ cursor.reset(new Cursor(rsp.GetQueryId()));
+
+ rowsAffectedIdx = 0;
return SqlResult::AI_SUCCESS;
}
@@ -308,10 +336,55 @@ namespace ignite
return SqlResult::AI_ERROR;
}
+ LOG_MSG("Page size: " << resultPage->GetSize());
+ LOG_MSG("Page is last: " << resultPage->IsLast());
+
cursor->UpdateData(resultPage);
return SqlResult::AI_SUCCESS;
}
+
+ SqlResult::Type DataQuery::MakeRequestMoreResults()
+ {
+ std::auto_ptr<ResultPage> resultPage(new ResultPage());
+
+ QueryMoreResultsRequest req(cursor->GetQueryId(), connection.GetConfiguration().GetPageSize());
+ QueryMoreResultsResponse rsp(*resultPage);
+
+ try
+ {
+ connection.SyncMessage(req, rsp);
+ }
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
+ catch (const IgniteError& err)
+ {
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
+
+ return SqlResult::AI_ERROR;
+ }
+
+ if (rsp.GetStatus() != ResponseStatus::SUCCESS)
+ {
+ LOG_MSG("Error: " << rsp.GetError());
+
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
+
+ return SqlResult::AI_ERROR;
+ }
+
+ LOG_MSG("Page size: " << resultPage->GetSize());
+ LOG_MSG("Page is last: " << resultPage->IsLast());
+
+ cachedNextPage = resultPage;
+ cursor.reset(new Cursor(rsp.GetQueryId()));
+
+ return SqlResult::AI_SUCCESS;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/foreign_keys_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/foreign_keys_query.cpp b/modules/platforms/cpp/odbc/src/query/foreign_keys_query.cpp
index 4ca7709..c22a3aa 100644
--- a/modules/platforms/cpp/odbc/src/query/foreign_keys_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/foreign_keys_query.cpp
@@ -125,6 +125,11 @@ namespace ignite
{
return 0;
}
+
+ SqlResult::Type ForeignKeysQuery::NextResultSet()
+ {
+ return SqlResult::AI_NO_DATA;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/primary_keys_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/primary_keys_query.cpp b/modules/platforms/cpp/odbc/src/query/primary_keys_query.cpp
index ef99db3..bb6f908 100644
--- a/modules/platforms/cpp/odbc/src/query/primary_keys_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/primary_keys_query.cpp
@@ -207,6 +207,11 @@ namespace ignite
{
return 0;
}
+
+ SqlResult::Type PrimaryKeysQuery::NextResultSet()
+ {
+ return SqlResult::AI_NO_DATA;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/special_columns_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/special_columns_query.cpp b/modules/platforms/cpp/odbc/src/query/special_columns_query.cpp
index b0f534c..01c7b96 100644
--- a/modules/platforms/cpp/odbc/src/query/special_columns_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/special_columns_query.cpp
@@ -116,6 +116,11 @@ namespace ignite
{
return 0;
}
+
+ SqlResult::Type SpecialColumnsQuery::NextResultSet()
+ {
+ return SqlResult::AI_NO_DATA;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp b/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
index 93f1f79..53fe49d 100644
--- a/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
@@ -215,6 +215,11 @@ namespace ignite
return 0;
}
+ SqlResult::Type TableMetadataQuery::NextResultSet()
+ {
+ return SqlResult::AI_NO_DATA;
+ }
+
SqlResult::Type TableMetadataQuery::MakeRequestGetTablesMeta()
{
QueryGetTablesMetaRequest req(catalog, schema, table, tableType);
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/query/type_info_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/type_info_query.cpp b/modules/platforms/cpp/odbc/src/query/type_info_query.cpp
index b4efca0..939458a 100644
--- a/modules/platforms/cpp/odbc/src/query/type_info_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/type_info_query.cpp
@@ -401,6 +401,11 @@ namespace ignite
{
return 0;
}
+
+ SqlResult::Type TypeInfoQuery::NextResultSet()
+ {
+ return SqlResult::AI_NO_DATA;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/87e37975/modules/platforms/cpp/odbc/src/statement.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index 36c1a0b..b167d44 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -824,19 +824,21 @@ namespace ignite
return currentQuery.get() && currentQuery->DataAvailable();
}
- void Statement::NextResults()
+ void Statement::MoreResults()
{
- IGNITE_ODBC_API_CALL(InternalNextResults());
+ IGNITE_ODBC_API_CALL(InternalMoreResults());
}
- SqlResult::Type Statement::InternalNextResults()
+ SqlResult::Type Statement::InternalMoreResults()
{
if (!currentQuery.get())
- return SqlResult::AI_NO_DATA;
+ {
+ AddStatusRecord(SqlState::SHY010_SEQUENCE_ERROR, "Query is not executed.");
- SqlResult::Type result = currentQuery->Close();
+ return SqlResult::AI_ERROR;
+ }
- return result == SqlResult::AI_SUCCESS ? SqlResult::AI_NO_DATA : result;
+ return currentQuery->NextResultSet();
}
void Statement::GetColumnAttribute(uint16_t colIdx, uint16_t attrId,
[13/19] ignite git commit: IGNITE-6690 DiscoverySpi: Clientmode
Ignite should not fail on handshake. This closes #2923.
Posted by ag...@apache.org.
IGNITE-6690 DiscoverySpi: Clientmode Ignite should not fail on handshake. This closes #2923.
Signed-off-by: nikolay_tikhonov <nt...@gridgain.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/035dcb7f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/035dcb7f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/035dcb7f
Branch: refs/heads/ignite-6778
Commit: 035dcb7f7163384648b2d01134d9e40fbb806bcf
Parents: 87e3797
Author: Alexey Popov <ta...@gmail.com>
Authored: Tue Oct 31 14:33:58 2017 +0300
Committer: nikolay_tikhonov <nt...@gridgain.com>
Committed: Tue Oct 31 14:35:29 2017 +0300
----------------------------------------------------------------------
.../ignite/spi/discovery/tcp/ClientImpl.java | 14 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 14 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 22 ++
.../tcp/TcpDiscoveryWithWrongServerTest.java | 332 +++++++++++++++++++
.../IgniteSpiDiscoverySelfTestSuite.java | 3 +
5 files changed, 377 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/035dcb7f/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 139c110..c9a4a5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -702,11 +702,17 @@ class ClientImpl extends TcpDiscoveryImpl {
}
if (X.hasCause(e, StreamCorruptedException.class)) {
- if (--sslConnectAttempts == 0)
- throw new IgniteSpiException("Unable to establish plain connection. " +
- "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+ // StreamCorruptedException could be caused by remote node failover
+ if (connectAttempts < 2) {
+ connectAttempts++;
- continue;
+ continue;
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Connect failed with StreamCorruptedException, skip address: " + addr);
+
+ break;
}
if (timeoutHelper.checkFailureTimeoutReached(e))
http://git-wip-us.apache.org/repos/asf/ignite/blob/035dcb7f/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 1c3ec2e..57d2faf 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1272,11 +1272,17 @@ class ServerImpl extends TcpDiscoveryImpl {
}
if (X.hasCause(e, StreamCorruptedException.class)) {
- if (--sslConnectAttempts == 0)
- throw new IgniteException("Unable to establish plain connection. " +
- "Was remote cluster configured with SSL? [rmtAddr=" + addr + ", errMsg=\"" + e.getMessage() + "\"]", e);
+ // StreamCorruptedException could be caused by remote node failover
+ if (connectAttempts < 2) {
+ connectAttempts++;
- continue;
+ continue;
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Connect failed with StreamCorruptedException, skip address: " + addr);
+
+ break;
}
if (timeoutHelper.checkFailureTimeoutReached(e))
http://git-wip-us.apache.org/repos/asf/ignite/blob/035dcb7f/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 922e787..3b83b2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
+import java.io.StreamCorruptedException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
@@ -40,7 +41,9 @@ import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLException;
import javax.net.ssl.SSLServerSocketFactory;
import javax.net.ssl.SSLSocketFactory;
import org.apache.ignite.Ignite;
@@ -265,6 +268,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
/** Maximum ack timeout value for receiving message acknowledgement in milliseconds (value is <tt>600,000ms</tt>). */
public static final long DFLT_MAX_ACK_TIMEOUT = 10 * 60 * 1000;
+ /** Ssl message pattern for StreamCorruptedException. */
+ private static Pattern sslMsgPattern = Pattern.compile("invalid stream header: 150\\d0\\d00");
+
/** Local address. */
protected String locAddr;
@@ -1616,6 +1622,22 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
"long GC pauses on remote node) [curTimeout=" + timeout +
", rmtAddr=" + sock.getRemoteSocketAddress() + ", rmtPort=" + sock.getPort() + ']');
+ StreamCorruptedException streamCorruptedCause = X.cause(e, StreamCorruptedException.class);
+
+ if (streamCorruptedCause != null) {
+ // Lets check StreamCorruptedException for SSL Alert message
+ // Sample SSL Alert message: 15:03:03:00:02:02:0a
+ // 15 = Alert
+ // 03:03 = SSL version
+ // 00:02 = payload length
+ // 02:0a = critical (02) / unexpected message (0a)
+ // So, check message for "invalid stream header: 150X0X00"
+
+ String msg = streamCorruptedCause.getMessage();
+
+ if (msg != null && sslMsgPattern.matcher(msg).matches())
+ streamCorruptedCause.initCause(new SSLException("Detected SSL alert in StreamCorruptedException"));
+ }
throw e;
}
finally {
http://git-wip-us.apache.org/repos/asf/ignite/blob/035dcb7f/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
new file mode 100644
index 0000000..768f5f7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryWithWrongServerTest.java
@@ -0,0 +1,332 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestThread;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Client-based discovery SPI test with non-Ignite servers.
+ */
+public class TcpDiscoveryWithWrongServerTest extends GridCommonAbstractTest {
+ /** Non-Ignite Server port. */
+ private final static int SERVER_PORT = 47500;
+
+ /** Non-Ignite Server socket. */
+ private ServerSocket srvSock;
+
+ /** Count of accepted connections to non-Ignite Server. */
+ private int connCnt;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder();
+
+ ipFinder.setAddresses(Collections.singleton("127.0.0.1:" + Integer.toString(SERVER_PORT) + ".." +
+ Integer.toString(SERVER_PORT + 2)));
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpiWithOrderedIps().setIpFinder(ipFinder));
+
+ if (igniteInstanceName.startsWith("client"))
+ cfg.setClientMode(true);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopTcpThread();
+
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * Starts tcp test thread
+ * @param workerFactory one of WorkerFactory
+ */
+ private void startTcpThread(final WorkerFactory workerFactory) {
+ connCnt = 0;
+
+ try {
+ srvSock = new ServerSocket(SERVER_PORT, 10, InetAddress.getByName("127.0.0.1"));
+ }
+ catch (Exception e) {
+ fail("Unexpected TcpServer exception " + e.getMessage());
+ }
+
+ new GridTestThread(new Runnable() {
+ @Override public void run() {
+ try {
+ while(!Thread.currentThread().isInterrupted()) {
+ Socket clientSock = srvSock.accept();
+
+ connCnt++;
+
+ // Create a new thread for socket connection.
+ new GridTestThread(workerFactory.newWorker(clientSock)).start();
+ }
+ }
+ catch (Exception e) {
+ if (!srvSock.isClosed())
+ e.printStackTrace();
+ }
+ }
+ }).start();
+ }
+
+ /**
+ * Stops tcp test thread
+ * @throws IOException IOException
+ */
+ private void stopTcpThread() throws IOException {
+ if (srvSock != null && !srvSock.isClosed())
+ srvSock.close();
+ }
+
+ /**
+ * Test that Client successfully ignores wrong responses during Discovery Handshake Procedure.
+ *
+ * @throws Exception in case of error.
+ */
+ public void testWrongHandshakeResponse() throws Exception {
+ startTcpThread(new SomeResponseWorker());
+
+ simpleTest();
+ }
+
+ /**
+ * Test that Client successfully ignores wrong responses during Discovery Handshake Procedure.
+ *
+ * @throws Exception in case of error.
+ */
+ public void testNoHandshakeResponse() throws Exception {
+ startTcpThread(new NoResponseWorker());
+
+ simpleTest();
+ }
+
+ /**
+ * Test that Client successfully ignores when server closes sockets after Discovery Handshake Request.
+ *
+ * @throws Exception in case of error.
+ */
+ public void testDisconnectOnRequest() throws Exception {
+ startTcpThread(new DisconnectOnRequestWorker());
+
+ simpleTest();
+ }
+
+ /**
+ * Test that Client successfully ignores when server closes sockets immediately.
+ *
+ * @throws Exception in case of error.
+ */
+ public void testEarlyDisconnect() throws Exception {
+ startTcpThread(new EarlyDisconnectWorker());
+
+ simpleTest();
+ }
+
+ /**
+ * Some simple sanity check with the Server and Client
+ * It is expected that both client and server could successfully perform Discovery Procedure when there is
+ * unknown (test) server in the ipFinder list.
+ */
+ private void simpleTest() {
+ try {
+ Ignite srv = startGrid("server");
+ Ignite client = startGrid("client");
+
+ awaitPartitionMapExchange();
+
+ assertEquals(2, srv.cluster().nodes().size());
+ assertEquals(2, client.cluster().nodes().size());
+ assertTrue(connCnt >= 2);
+
+ srv.getOrCreateCache(DEFAULT_CACHE_NAME).put(1, 1);
+
+ assertEquals(1, client.getOrCreateCache(DEFAULT_CACHE_NAME).get(1));
+ }
+ catch (Exception e) {
+ fail("Failed with unexpected exception: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Just a factory for runnable workers
+ */
+ private interface WorkerFactory {
+ /**
+ * Creates a new worker for socket
+ * @param clientSock socket for worker
+ * @return runnable Worker
+ */
+ Runnable newWorker(Socket clientSock);
+ }
+
+ /**
+ * SocketWorker
+ */
+ private abstract class SocketWorker implements Runnable {
+ /** Client socket. */
+ Socket clientSock;
+
+ /**
+ * @param clientSock Client socket.
+ */
+ SocketWorker(Socket clientSock) {
+ this.clientSock = clientSock;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ InputStream input = clientSock.getInputStream();
+ OutputStream output = clientSock.getOutputStream();
+ byte[] buf = new byte[1024];
+
+ while (!clientSock.isClosed() && input.read(buf) > 0)
+ action(input, output);
+
+ if (!clientSock.isClosed())
+ clientSock.close();
+ }
+ catch (IOException e) {
+ log.error("Unexpected error", e);
+ }
+ }
+
+ /**
+ * @param input socket input stream
+ * @param output socket output stream
+ * @throws IOException IOException
+ */
+ public abstract void action(InputStream input, OutputStream output) throws IOException;
+ }
+
+ /**
+ * SomeResponseWorker.
+ */
+ private class SomeResponseWorker implements WorkerFactory {
+ /** {@inheritDoc} */
+ @Override public Runnable newWorker(Socket clientSock) {
+ return new SocketWorker(clientSock) {
+ @Override public void action(InputStream input, OutputStream output) throws IOException {
+ output.write("Some response".getBytes());
+
+ log.error("TEST: Some response was sent to " + clientSock.getRemoteSocketAddress());
+ }
+ };
+ }
+ }
+
+ /**
+ * NoResponseWorker.
+ */
+ private class NoResponseWorker implements WorkerFactory {
+ /** {@inheritDoc} */
+ @Override public Runnable newWorker(Socket clientSock) {
+ return new SocketWorker(clientSock) {
+ @Override public void action(InputStream input, OutputStream output) throws IOException {
+ log.error("TEST: No response was sent to " + clientSock.getRemoteSocketAddress());
+ }
+ };
+ }
+ }
+
+ /**
+ * DisconnectOnRequestWorker.
+ */
+ private class DisconnectOnRequestWorker implements WorkerFactory {
+ /** {@inheritDoc} */
+ @Override public Runnable newWorker(Socket clientSock) {
+ return new SocketWorker(clientSock) {
+ @Override public void action(InputStream input, OutputStream output) throws IOException {
+ clientSock.close();
+
+ log.error("TEST: Socket closed for " + clientSock.getRemoteSocketAddress());
+ }
+ };
+ }
+ }
+
+ /**
+ * EarlyDisconnectWorker.
+ */
+ private class EarlyDisconnectWorker implements WorkerFactory {
+ /** {@inheritDoc} */
+ @Override public Runnable newWorker(Socket clientSock) {
+ return new SocketWorker(clientSock) {
+ @Override public void action(InputStream input, OutputStream output) throws IOException {
+ // No-op
+ }
+
+ @Override public void run() {
+ try {
+ clientSock.close();
+
+ log.error("TEST: Socket closed for " + clientSock.getRemoteSocketAddress());
+ }
+ catch (IOException e) {
+ log.error("Unexpected error", e);
+ }
+ }
+ };
+ }
+ }
+
+ /**
+ * TcpDiscoverySpi with non-shuffled resolved IP addresses. We should ensure that in this test non-Ignite server
+ * is the first element of the addresses list
+ */
+ class TcpDiscoverySpiWithOrderedIps extends TcpDiscoverySpi {
+ /** {@inheritDoc} */
+ @Override protected Collection<InetSocketAddress> resolvedAddresses() throws IgniteSpiException {
+ Collection<InetSocketAddress> shuffled = super.resolvedAddresses();
+ List<InetSocketAddress> res = new ArrayList<>(shuffled);
+
+ Collections.sort(res, new Comparator<InetSocketAddress>() {
+ @Override public int compare(InetSocketAddress o1, InetSocketAddress o2) {
+ return o1.toString().compareTo(o2.toString());
+ }
+ });
+
+ return res;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/035dcb7f/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index ff4c9c1..626875c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -43,6 +43,7 @@ import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslTrustedSelfTest;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslTrustedUntrustedTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryWithWrongServerTest;
import org.apache.ignite.spi.discovery.tcp.ipfinder.jdbc.TcpDiscoveryJdbcIpFinderSelfTest;
import org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinderSelfTest;
import org.apache.ignite.spi.discovery.tcp.ipfinder.sharedfs.TcpDiscoverySharedFsIpFinderSelfTest;
@@ -95,6 +96,8 @@ public class IgniteSpiDiscoverySelfTestSuite extends TestSuite {
suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
suite.addTest(new TestSuite(AuthenticationRestartTest.class));
+ suite.addTest(new TestSuite(TcpDiscoveryWithWrongServerTest.class));
+
// Client connect.
suite.addTest(new TestSuite(IgniteClientConnectTest.class));
suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
[18/19] ignite git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-6778
Posted by ag...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-6778
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/54ccc9fe
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/54ccc9fe
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/54ccc9fe
Branch: refs/heads/ignite-6778
Commit: 54ccc9fe763b1588814356a97ae9d13de53d4d6d
Parents: 6d47417 2f468d8
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Nov 1 12:28:58 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Nov 1 12:28:58 2017 +0300
----------------------------------------------------------------------
.../wal/reader/StandaloneGridKernalContext.java | 20 +-
.../processors/odbc/odbc/OdbcMessageParser.java | 60 +++-
.../odbc/odbc/OdbcQueryExecuteBatchResult.java | 20 +-
.../odbc/odbc/OdbcQueryExecuteResult.java | 11 +-
.../odbc/odbc/OdbcQueryMoreResultsRequest.java | 61 ++++
.../odbc/odbc/OdbcQueryMoreResultsResult.java | 66 ++++
.../processors/odbc/odbc/OdbcQueryResults.java | 106 ++++++
.../processors/odbc/odbc/OdbcRequest.java | 3 +
.../odbc/odbc/OdbcRequestHandler.java | 175 +++++-----
.../processors/odbc/odbc/OdbcResultSet.java | 101 ++++++
.../processors/odbc/odbc/OdbcUtils.java | 31 ++
.../ignite/spi/discovery/tcp/ClientImpl.java | 14 +-
.../ignite/spi/discovery/tcp/ServerImpl.java | 14 +-
.../spi/discovery/tcp/TcpDiscoverySpi.java | 22 ++
.../db/wal/reader/IgniteWalReaderTest.java | 283 ++++++++++------
.../tcp/TcpDiscoveryWithWrongServerTest.java | 332 +++++++++++++++++++
.../IgniteSpiDiscoverySelfTestSuite.java | 3 +
modules/docker/2.3.0/Dockerfile | 46 +++
modules/docker/2.3.0/run.sh | 51 +++
modules/docker/Dockerfile | 2 +-
.../math/distributed/keys/BlockMatrixKey.java | 12 +-
.../distributed/keys/impl/BlockMatrixKey.java | 49 ++-
.../matrix/SparseBlockDistributedMatrix.java | 15 +-
.../impls/matrix/SparseDistributedMatrix.java | 2 +-
.../storage/matrix/BlockMatrixStorage.java | 213 ++++++------
.../SparseDistributedBlockMatrixTest.java | 15 +-
.../examples/odbc-example/src/odbc_example.cpp | 8 +-
.../cpp/odbc-test/src/queries_test.cpp | 200 ++++++++++-
.../cpp/odbc/include/ignite/odbc/message.h | 97 +++++-
.../include/ignite/odbc/query/batch_query.h | 16 +-
.../ignite/odbc/query/column_metadata_query.h | 7 +
.../odbc/include/ignite/odbc/query/data_query.h | 22 +-
.../ignite/odbc/query/foreign_keys_query.h | 7 +
.../ignite/odbc/query/primary_keys_query.h | 7 +
.../cpp/odbc/include/ignite/odbc/query/query.h | 7 +
.../ignite/odbc/query/special_columns_query.h | 7 +
.../ignite/odbc/query/table_metadata_query.h | 7 +
.../include/ignite/odbc/query/type_info_query.h | 7 +
.../cpp/odbc/include/ignite/odbc/statement.h | 6 +-
modules/platforms/cpp/odbc/src/cursor.cpp | 2 +-
modules/platforms/cpp/odbc/src/message.cpp | 62 +++-
modules/platforms/cpp/odbc/src/odbc.cpp | 2 +-
.../cpp/odbc/src/query/batch_query.cpp | 80 ++---
.../odbc/src/query/column_metadata_query.cpp | 5 +
.../platforms/cpp/odbc/src/query/data_query.cpp | 93 +++++-
.../cpp/odbc/src/query/foreign_keys_query.cpp | 5 +
.../cpp/odbc/src/query/primary_keys_query.cpp | 5 +
.../odbc/src/query/special_columns_query.cpp | 5 +
.../cpp/odbc/src/query/table_metadata_query.cpp | 5 +
.../cpp/odbc/src/query/type_info_query.cpp | 5 +
modules/platforms/cpp/odbc/src/statement.cpp | 14 +-
.../Cache/DataStorageMetricsTest.cs | 2 +-
.../Apache.Ignite.Core.csproj | 4 -
53 files changed, 1931 insertions(+), 483 deletions(-)
----------------------------------------------------------------------
[08/19] ignite git commit: IGNITE-6793 More debug output
Posted by ag...@apache.org.
IGNITE-6793 More debug output
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0134f2b4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0134f2b4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0134f2b4
Branch: refs/heads/ignite-6778
Commit: 0134f2b4447c97c0bf3ddb4fac3b423248030eec
Parents: 58dbbc5
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Oct 30 18:18:53 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Oct 30 18:19:36 2017 +0300
----------------------------------------------------------------------
.../dht/preloader/InitNewCoordinatorFuture.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0134f2b4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
index b5acd4b..596fa8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
@@ -38,6 +38,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
@@ -113,6 +114,10 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
nodes.add(node);
}
+ else if (!node.isLocal()) {
+ if (log.isInfoEnabled())
+ log.info("Init new coordinator future will skip remote node: " + node);
+ }
}
if (exchFut.context().mergeExchanges() && !curDiscoCache.version().equals(discoCache.version())) {
@@ -148,8 +153,10 @@ public class InitNewCoordinatorFuture extends GridCompoundFuture {
}
if (log.isInfoEnabled()) {
- log.info("Try restore exchange result [allNodes=" + awaited +
- ", joined=" + joinedNodes.keySet() + ']');
+ log.info("Try restore exchange result [awaited=" + awaited +
+ ", joined=" + joinedNodes.keySet() +
+ ", nodes=" + U.nodeIds(nodes) +
+ ", discoAllNodes=" + U.nodeIds(discoCache.allNodes()) + ']');
}
if (!nodes.isEmpty()) {
[10/19] ignite git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/ignite into ignite-6778
Posted by ag...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ignite into ignite-6778
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6d474174
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6d474174
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6d474174
Branch: refs/heads/ignite-6778
Commit: 6d474174cd6a6e32c8f61bbe51f627f38057f00d
Parents: 41ff5ff 4f157f6
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Oct 30 18:39:35 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Oct 30 18:39:35 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/pagemem/PageIdUtils.java | 19 +++
.../processors/cache/GridCacheUtils.java | 14 +-
.../dht/preloader/InitNewCoordinatorFuture.java | 11 +-
.../IgniteCacheDatabaseSharedManager.java | 26 +--
.../persistence/freelist/FreeListImpl.java | 18 +-
.../cache/persistence/tree/io/DataPageIO.java | 17 +-
.../internal/visor/VisorDataTransferObject.java | 3 +
.../node/VisorClientConnectorConfiguration.java | 166 +++++++++++++++++++
.../visor/node/VisorGridConfiguration.java | 26 ++-
.../node/VisorSqlConnectorConfiguration.java | 3 +
.../cache/CacheRebalancingSelfTest.java | 2 +
.../testsuites/IgniteReproducingSuite.java | 42 +++++
.../ml/math/decompositions/LUDecomposition.java | 2 +-
.../config/VisorConfigurationCommand.scala | 29 +++-
modules/web-console/backend/app/mongo.js | 11 ++
.../generator/AbstractTransformer.js | 5 +
.../generator/ConfigurationGenerator.js | 35 +++-
.../frontend/app/modules/sql/sql.controller.js | 2 +-
.../configuration/clusters/client-connector.pug | 59 +++++++
.../configuration/clusters/sql-connector.pug | 4 +-
.../frontend/app/utils/SimpleWorkerPool.js | 1 +
.../views/configuration/clusters.tpl.pug | 6 +
.../ignite/console/demo/AgentClusterDemo.java | 32 ++--
.../demo/service/DemoCachesLoadService.java | 3 +-
24 files changed, 484 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6d474174/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
[17/19] ignite git commit: .NET: Fix DataStorageMetricsTest
Posted by ag...@apache.org.
.NET: Fix DataStorageMetricsTest
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2f468d87
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2f468d87
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2f468d87
Branch: refs/heads/ignite-6778
Commit: 2f468d87098a5545fc3960bae9d73d0b9a79077a
Parents: 5542c70
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Wed Nov 1 09:44:07 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Wed Nov 1 09:44:07 2017 +0300
----------------------------------------------------------------------
.../Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2f468d87/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
index b24c20b..1bc0218 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/DataStorageMetricsTest.cs
@@ -79,7 +79,7 @@ namespace Apache.Ignite.Core.Tests.Cache
Assert.AreEqual(0, metrics.WalArchiveSegments);
Assert.AreEqual(0, metrics.WalFsyncTimeAverage);
- Assert.AreEqual(77, metrics.LastCheckpointTotalPagesNumber);
+ Assert.AreEqual(89, metrics.LastCheckpointTotalPagesNumber);
Assert.AreEqual(10, metrics.LastCheckpointDataPagesNumber);
Assert.AreEqual(0, metrics.LastCheckpointCopiedOnWritePagesNumber);
Assert.AreEqual(TimeSpan.Zero, metrics.LastCheckpointLockWaitDuration);
[11/19] ignite git commit: .NET: Fix unmodified project rebuild
Posted by ag...@apache.org.
.NET: Fix unmodified project rebuild
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/61621b88
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/61621b88
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/61621b88
Branch: refs/heads/ignite-6778
Commit: 61621b88d4b3e9e4304974707f998bc126a97eef
Parents: 4f157f6
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Oct 30 19:14:28 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Oct 30 19:14:28 2017 +0300
----------------------------------------------------------------------
.../dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/61621b88/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index 4e73d15..852c071 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -613,10 +613,6 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>
- <ItemGroup>
- <Folder Include="Data\" />
- <Folder Include="Impl\Common\JavaObjects\" />
- </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup Condition="'$(Platform)' == 'AnyCPU'">
<PreBuildEvent>cd "$(ProjectDir)"
[09/19] ignite git commit: IGNITE-6746 Persist system cache when
persistent data regions are present
Posted by ag...@apache.org.
IGNITE-6746 Persist system cache when persistent data regions are present
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4f157f65
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4f157f65
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4f157f65
Branch: refs/heads/ignite-6778
Commit: 4f157f6545122a4b238edd85bdaa1eeea9a6a88a
Parents: 0134f2b
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Mon Oct 30 18:31:59 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Oct 30 18:31:59 2017 +0300
----------------------------------------------------------------------
.../processors/cache/GridCacheUtils.java | 14 ++++++++---
.../IgniteCacheDatabaseSharedManager.java | 26 ++++++++++++--------
2 files changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4f157f65/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 26e2254..1f289d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1701,14 +1701,22 @@ public class GridCacheUtils {
return false;
}
+
/**
* @return {@code true} if persistence is enabled for at least one data region, {@code false} if not.
*/
public static boolean isPersistenceEnabled(IgniteConfiguration cfg) {
- if (cfg.getDataStorageConfiguration() == null)
+ return isPersistenceEnabled(cfg.getDataStorageConfiguration());
+ }
+
+ /**
+ * @return {@code true} if persistence is enabled for at least one data region, {@code false} if not.
+ */
+ public static boolean isPersistenceEnabled(DataStorageConfiguration cfg) {
+ if (cfg == null)
return false;
- DataRegionConfiguration dfltReg = cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration();
+ DataRegionConfiguration dfltReg = cfg.getDefaultDataRegionConfiguration();
if (dfltReg == null)
return false;
@@ -1716,7 +1724,7 @@ public class GridCacheUtils {
if (dfltReg.isPersistenceEnabled())
return true;
- DataRegionConfiguration[] regCfgs = cfg.getDataStorageConfiguration().getDataRegionConfigurations();
+ DataRegionConfiguration[] regCfgs = cfg.getDataRegionConfigurations();
if (regCfgs == null)
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/4f157f65/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 933c195..f00edbe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -57,6 +57,7 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseL
import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
@@ -218,36 +219,35 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
if (dataRegionCfgs != null) {
for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs)
- addDataRegion(memCfg, dataRegionCfg, dataRegionCfg.getName());
+ addDataRegion(memCfg, dataRegionCfg);
}
addDataRegion(
memCfg,
- memCfg.getDefaultDataRegionConfiguration(),
- memCfg.getDefaultDataRegionConfiguration().getName()
+ memCfg.getDefaultDataRegionConfiguration()
);
addDataRegion(
memCfg,
createSystemDataRegion(
memCfg.getSystemRegionInitialSize(),
- memCfg.getSystemRegionMaxSize()
- ),
- SYSTEM_DATA_REGION_NAME
+ memCfg.getSystemRegionMaxSize(),
+ CU.isPersistenceEnabled(memCfg)
+ )
);
}
/**
* @param dataStorageCfg Database config.
* @param dataRegionCfg Data region config.
- * @param dataRegionName Data region name.
* @throws IgniteCheckedException If failed to initialize swap path.
*/
private void addDataRegion(
DataStorageConfiguration dataStorageCfg,
- DataRegionConfiguration dataRegionCfg,
- String dataRegionName
+ DataRegionConfiguration dataRegionCfg
) throws IgniteCheckedException {
+ String dataRegionName = dataRegionCfg.getName();
+
String dfltMemPlcName = dataStorageCfg.getDefaultDataRegionConfiguration().getName();
if (dfltMemPlcName == null)
@@ -315,15 +315,21 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
/**
* @param sysCacheInitSize Initial size of PageMemory to be created for system cache.
* @param sysCacheMaxSize Maximum size of PageMemory to be created for system cache.
+ * @param persistenceEnabled Persistence enabled flag.
*
* @return {@link DataRegionConfiguration configuration} of DataRegion for system cache.
*/
- private DataRegionConfiguration createSystemDataRegion(long sysCacheInitSize, long sysCacheMaxSize) {
+ private DataRegionConfiguration createSystemDataRegion(
+ long sysCacheInitSize,
+ long sysCacheMaxSize,
+ boolean persistenceEnabled
+ ) {
DataRegionConfiguration res = new DataRegionConfiguration();
res.setName(SYSTEM_DATA_REGION_NAME);
res.setInitialSize(sysCacheInitSize);
res.setMaxSize(sysCacheMaxSize);
+ res.setPersistenceEnabled(persistenceEnabled);
return res;
}
[06/19] ignite git commit: IGNITE-6713 Fixed flaky rebalance test -
Fixes #2914.
Posted by ag...@apache.org.
IGNITE-6713 Fixed flaky rebalance test - Fixes #2914.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/555b71ed
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/555b71ed
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/555b71ed
Branch: refs/heads/ignite-6778
Commit: 555b71ed071abfeb84d1faa187776c61ca751858
Parents: 1d39507
Author: dpavlov <dp...@gridgain.com>
Authored: Mon Oct 30 15:24:20 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Mon Oct 30 15:24:20 2017 +0300
----------------------------------------------------------------------
.../cache/CacheRebalancingSelfTest.java | 2 +
.../testsuites/IgniteReproducingSuite.java | 42 ++++++++++++++++++++
2 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/555b71ed/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
index e5a836a..147b76a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalancingSelfTest.java
@@ -23,6 +23,7 @@ import java.util.Set;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
@@ -51,6 +52,7 @@ public class CacheRebalancingSelfTest extends GridCommonAbstractTest {
CacheConfiguration<Integer,Integer> rebalabceCacheCfg = new CacheConfiguration<>();
rebalabceCacheCfg.setBackups(1);
rebalabceCacheCfg.setName(REBALANCE_TEST_CACHE_NAME);
+ rebalabceCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME), rebalabceCacheCfg);
http://git-wip-us.apache.org/repos/asf/ignite/blob/555b71ed/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteReproducingSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteReproducingSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteReproducingSuite.java
new file mode 100644
index 0000000..22bd610
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteReproducingSuite.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+
+/**
+ * Test suite for cycled run tests on PR code. <br>
+ * This empty suite may be used in case it is needed to run
+ * some test subset to reproduce an issue.<br>
+ *
+ * You may launch and check results on
+ * https://ci.ignite.apache.org/viewType.html?buildTypeId=Ignite20Tests_IgniteReproducingSuite
+ *
+ * This suite is not included into main build
+ */
+public class IgniteReproducingSuite extends TestSuite {
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Issue Reproducing Test Suite");
+
+ //uncomment to add some test
+ //for (int i = 0; i < 100; i++)
+ // suite.addTestSuite();
+
+ return suite;
+ }
+}
[14/19] ignite git commit: IGNITE-6761: Change block numeration in
SparseBlockMatrixStorage.
Posted by ag...@apache.org.
IGNITE-6761: Change block numeration in SparseBlockMatrixStorage.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/94dd19d3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/94dd19d3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/94dd19d3
Branch: refs/heads/ignite-6778
Commit: 94dd19d36812d0bea3a1652ff745fbefc10eba70
Parents: 035dcb7
Author: Yury Babak <yb...@gridgain.com>
Authored: Tue Oct 31 15:35:28 2017 +0300
Committer: Igor Sapego <is...@gridgain.com>
Committed: Tue Oct 31 15:37:40 2017 +0300
----------------------------------------------------------------------
.../math/distributed/keys/BlockMatrixKey.java | 12 +-
.../distributed/keys/impl/BlockMatrixKey.java | 49 +++--
.../matrix/SparseBlockDistributedMatrix.java | 15 +-
.../impls/matrix/SparseDistributedMatrix.java | 2 +-
.../storage/matrix/BlockMatrixStorage.java | 213 +++++++++----------
.../SparseDistributedBlockMatrixTest.java | 15 +-
6 files changed, 157 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dd19d3/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
index c55b950..091b325 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/BlockMatrixKey.java
@@ -17,14 +17,22 @@
package org.apache.ignite.ml.math.distributed.keys;
+import org.apache.ignite.internal.util.lang.IgnitePair;
import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
/**
* Cache key for blocks in {@link SparseBlockDistributedMatrix}.
+ *
+ * TODO: check if using {@link IgnitePair} will be better for block id.
*/
public interface BlockMatrixKey extends MatrixCacheKey {
/**
- * @return block id.
+ * @return block row id.
+ */
+ public long blockRowId();
+
+ /**
+ * @return block col id.
*/
- public long blockId();
+ public long blockColId();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dd19d3/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
index 5fd1a16..2edd9cb 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/distributed/keys/impl/BlockMatrixKey.java
@@ -42,8 +42,10 @@ import org.jetbrains.annotations.Nullable;
public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.keys.BlockMatrixKey, Externalizable, Binarylizable {
/** */
private static final long serialVersionUID = 0L;
- /** Block ID */
- private long blockId;
+ /** Block row ID */
+ private long blockIdRow;
+ /** Block col ID */
+ private long blockIdCol;
/** Matrix ID */
private IgniteUuid matrixUuid;
/** Block affinity key. */
@@ -59,22 +61,28 @@ public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.key
/**
* Construct matrix block key.
*
- * @param blockId Block id.
* @param matrixUuid Matrix uuid.
* @param affinityKey Affinity key.
*/
- public BlockMatrixKey(long blockId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) {
- assert blockId >= 0;
+ public BlockMatrixKey(long rowId, long colId, IgniteUuid matrixUuid, @Nullable IgniteUuid affinityKey) {
+ assert rowId >= 0;
+ assert colId >= 0;
assert matrixUuid != null;
- this.blockId = blockId;
+ this.blockIdRow = rowId;
+ this.blockIdCol = colId;
this.matrixUuid = matrixUuid;
this.affinityKey = affinityKey;
}
/** {@inheritDoc} */
- @Override public long blockId() {
- return blockId;
+ @Override public long blockRowId() {
+ return blockIdRow;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long blockColId() {
+ return blockIdCol;
}
/** {@inheritDoc} */
@@ -91,14 +99,16 @@ public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.key
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeGridUuid(out, matrixUuid);
U.writeGridUuid(out, affinityKey);
- out.writeLong(blockId);
+ out.writeLong(blockIdRow);
+ out.writeLong(blockIdCol);
}
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
matrixUuid = U.readGridUuid(in);
affinityKey = U.readGridUuid(in);
- blockId = in.readLong();
+ blockIdRow = in.readLong();
+ blockIdCol = in.readLong();
}
/** {@inheritDoc} */
@@ -107,7 +117,8 @@ public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.key
BinaryUtils.writeIgniteUuid(out, matrixUuid);
BinaryUtils.writeIgniteUuid(out, affinityKey);
- out.writeLong(blockId);
+ out.writeLong(blockIdRow);
+ out.writeLong(blockIdCol);
}
/** {@inheritDoc} */
@@ -116,12 +127,19 @@ public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.key
matrixUuid = BinaryUtils.readIgniteUuid(in);
affinityKey = BinaryUtils.readIgniteUuid(in);
- blockId = in.readLong();
+ blockIdRow = in.readLong();
+ blockIdCol = in.readLong();
}
/** {@inheritDoc} */
@Override public int hashCode() {
- return matrixUuid.hashCode() + (int)(blockId ^ (blockId >>> 32));
+ int res = 37;
+
+ res += res * 37 + blockIdCol;
+ res += res * 37 + blockIdRow;
+ res += res * 37 + matrixUuid.hashCode();
+
+ return res;
}
/** {@inheritDoc} */
@@ -134,11 +152,14 @@ public class BlockMatrixKey implements org.apache.ignite.ml.math.distributed.key
BlockMatrixKey that = (BlockMatrixKey)obj;
- return blockId == that.blockId && matrixUuid.equals(that.matrixUuid) && F.eq(affinityKey, that.affinityKey);
+ return blockIdRow == that.blockIdRow && blockIdCol == that.blockIdCol && matrixUuid.equals(that.matrixUuid)
+ && F.eq(affinityKey, that.affinityKey);
}
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(BlockMatrixKey.class, this);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dd19d3/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
index 1dcf1d8..3d542bc 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseBlockDistributedMatrix.java
@@ -25,6 +25,7 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.Affinity;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.lang.IgnitePair;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.StorageConstants;
@@ -106,7 +107,7 @@ public class SparseBlockDistributedMatrix extends AbstractMatrix implements Stor
CacheUtils.bcast(cacheName, () -> {
Ignite ignite = Ignition.localIgnite();
- Affinity affinity = ignite.affinity(cacheName);
+ Affinity<BlockMatrixKey> affinity = ignite.affinity(cacheName);
IgniteCache<BlockMatrixKey, BlockEntry> cache = ignite.getOrCreateCache(cacheName);
ClusterNode locNode = ignite.cluster().localNode();
@@ -122,11 +123,15 @@ public class SparseBlockDistributedMatrix extends AbstractMatrix implements Stor
// compute Cij locally on each node
// TODO: IGNITE:5114, exec in parallel
locKeys.forEach(key -> {
- long newBlockId = key.blockId();
+ long newBlockIdRow = key.blockRowId();
+ long newBlockIdCol = key.blockColId();
+
+ IgnitePair<Long> newBlockId = new IgnitePair<>(newBlockIdRow, newBlockIdCol);
+
BlockEntry blockC = null;
- List<BlockEntry> aRow = matrixA.storage().getRowForBlock(newBlockId, storageC);
- List<BlockEntry> bCol = matrixB.storage().getColForBlock(newBlockId, storageC);
+ List<BlockEntry> aRow = matrixA.storage().getRowForBlock(newBlockId);
+ List<BlockEntry> bCol = matrixB.storage().getColForBlock(newBlockId);
for (int i = 0; i < aRow.size(); i++) {
BlockEntry blockA = aRow.get(i);
@@ -137,7 +142,7 @@ public class SparseBlockDistributedMatrix extends AbstractMatrix implements Stor
blockC = blockC == null ? tmpBlock : new BlockEntry(blockC.plus(tmpBlock));
}
- cache.put(storageC.getCacheKey(newBlockId), blockC);
+ cache.put(storageC.getCacheKey(newBlockIdRow, newBlockIdCol), blockC);
});
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dd19d3/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
index 92d7c39..9a18f8b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedMatrix.java
@@ -119,7 +119,7 @@ public class SparseDistributedMatrix extends AbstractMatrix implements StorageCo
CacheUtils.bcast(cacheName, () -> {
Ignite ignite = Ignition.localIgnite();
- Affinity affinity = ignite.affinity(cacheName);
+ Affinity<RowColMatrixKey> affinity = ignite.affinity(cacheName);
IgniteCache<RowColMatrixKey, BlockEntry> cache = ignite.getOrCreateCache(cacheName);
ClusterNode locNode = ignite.cluster().localNode();
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dd19d3/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
index 0f285c2..0978e5a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/impls/storage/matrix/BlockMatrixStorage.java
@@ -24,8 +24,6 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.LongStream;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.CacheAtomicityMode;
@@ -33,6 +31,7 @@ import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CachePeekMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.lang.IgnitePair;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.MatrixStorage;
@@ -88,10 +87,8 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
this.rows = rows;
this.cols = cols;
- //cols % maxBlockEdge > 0 ? 1 : 0
-
- this.blocksInRow = cols % maxBlockEdge == 0 ? cols / maxBlockEdge : cols / maxBlockEdge + 1;
- this.blocksInCol = rows % maxBlockEdge == 0 ? rows / maxBlockEdge : rows / maxBlockEdge + 1;
+ this.blocksInRow = rows % maxBlockEdge == 0 ? rows / maxBlockEdge : rows / maxBlockEdge + 1;
+ this.blocksInCol = cols % maxBlockEdge == 0 ? cols / maxBlockEdge : cols / maxBlockEdge + 1;
cache = newCache();
@@ -135,6 +132,20 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
return RANDOM_ACCESS_MODE;
}
+ /**
+ * @return Blocks in column.
+ */
+ public int blocksInCol() {
+ return blocksInCol;
+ }
+
+ /**
+ * @return Blocks in row.
+ */
+ public int blocksInRow() {
+ return blocksInRow;
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
out.writeInt(rows);
@@ -182,37 +193,7 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
/** Delete all data from cache. */
@Override public void destroy() {
- long maxBlockId = getBlockId(cols, rows);
-
- Set<BlockMatrixKey> keyset = LongStream.rangeClosed(0, maxBlockId).mapToObj(this::getCacheKey).collect(Collectors.toSet());
-
- cache.clearAll(keyset);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = 1;
-
- res = res * 37 + cols;
- res = res * 37 + rows;
- res = res * 37 + uuid.hashCode();
- res = res * 37 + cache.hashCode();
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object obj) {
- if (this == obj)
- return true;
-
- if (obj == null || getClass() != obj.getClass())
- return false;
-
- BlockMatrixStorage that = (BlockMatrixStorage)obj;
-
- return rows == that.rows && cols == that.cols && uuid.equals(that.uuid)
- && (cache != null ? cache.equals(that.cache) : that.cache == null);
+ cache.clearAll(getAllKeys());
}
/**
@@ -225,126 +206,123 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
}
/**
- * Build the cache key for the given block id
+ * Build the cache key for the given blocks id.
+ *
+ * NB: NOT cell indices.
*/
- public BlockMatrixKey getCacheKey(long blockId) {
- return new BlockMatrixKey(blockId, uuid, getAffinityKey(blockId));
+ public BlockMatrixKey getCacheKey(long blockIdRow, long blockIdCol) {
+ return new BlockMatrixKey(blockIdRow, blockIdCol, uuid, getAffinityKey(blockIdRow, blockIdCol));
}
/**
- * Get rows for current block.
+ * Build the cache key for the given blocks id.
*
- * @param blockId block id.
- * @param storageC result storage.
- * @return The list of block entries.
+ * NB: NOT cell indices.
*/
- public List<BlockEntry> getRowForBlock(long blockId, BlockMatrixStorage storageC) {
- long blockRow = blockId / storageC.blocksInCol;
- long blockCol = blockId % storageC.blocksInRow;
+ public BlockMatrixKey getCacheKey(IgnitePair<Long> blockId) {
+ return new BlockMatrixKey(blockId.get1(), blockId.get2(), uuid, getAffinityKey(blockId.get1(), blockId.get2()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public Set<BlockMatrixKey> getAllKeys() {
+ IgnitePair<Long> maxBlockId = getBlockId(rows, cols);
+
+ Set<BlockMatrixKey> keyset = new HashSet<>();
- long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol);
+ for(int i = 0; i <= maxBlockId.get1(); i++)
+ for(int j = 0; j <= maxBlockId.get2(); j++)
+ keyset.add(getCacheKey(i,j));
- return getRowForBlock(locBlock);
+ return keyset;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String cacheName() {
+ return CACHE_NAME;
}
/**
- * Get cols for current block.
+ * Get rows for current block.
*
* @param blockId block id.
- * @param storageC result storage.
* @return The list of block entries.
*/
- public List<BlockEntry> getColForBlock(long blockId, BlockMatrixStorage storageC) {
- long blockRow = blockId / storageC.blocksInCol;
- long blockCol = blockId % storageC.blocksInRow;
+ public List<BlockEntry> getRowForBlock(IgnitePair<Long> blockId) {
+ List<BlockEntry> res = new LinkedList<>();
- long locBlock = this.blocksInRow * (blockRow) + (blockCol >= this.blocksInRow ? (blocksInRow - 1) : blockCol);
+ for (int i = 0; i < blocksInCol; i++)
+ res.add(getEntryById(new IgnitePair<>((long) i, blockId.get2())));
- return getColForBlock(locBlock);
+ return res;
}
- /** {@inheritDoc} */
- @Override public Set<BlockMatrixKey> getAllKeys() {
- long maxBlockId = numberOfBlocks();
- Set<BlockMatrixKey> keys = new HashSet<>();
+ /**
+ * Get cols for current block.
+ *
+ * @param blockId block id.
+ * @return The list of block entries.
+ */
+ public List<BlockEntry> getColForBlock(IgnitePair<Long> blockId) {
+ List<BlockEntry> res = new LinkedList<>();
- for (long id = 0; id < maxBlockId; id++)
- keys.add(getCacheKey(id));
+ for (int i = 0; i < blocksInRow; i++)
+ res.add(getEntryById(new IgnitePair<>(blockId.get1(), (long) i)));
- return keys;
+ return res;
}
/** {@inheritDoc} */
- @Override public String cacheName() {
- return CACHE_NAME;
- }
-
- /** */
- private List<BlockEntry> getRowForBlock(long blockId) {
- List<BlockEntry> res = new LinkedList<>();
-
- boolean isFirstRow = blockId < blocksInRow;
-
- long startBlock = isFirstRow ? 0 : blockId - blockId % blocksInRow;
- long endBlock = startBlock + blocksInRow - 1;
+ @Override public int hashCode() {
+ int res = 1;
- for (long i = startBlock; i <= endBlock; i++)
- res.add(getEntryById(i));
+ res = res * 37 + cols;
+ res = res * 37 + rows;
+ res = res * 37 + uuid.hashCode();
+ res = res * 37 + cache.hashCode();
return res;
}
- /** */
- private List<BlockEntry> getColForBlock(long blockId) {
- List<BlockEntry> res = new LinkedList<>();
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
- long startBlock = blockId % blocksInRow;
- long endBlock = startBlock + blocksInRow * (blocksInCol - 1);
+ if (obj == null || getClass() != obj.getClass())
+ return false;
- for (long i = startBlock; i <= endBlock; i += blocksInRow)
- res.add(getEntryById(i));
+ BlockMatrixStorage that = (BlockMatrixStorage)obj;
- return res;
+ return rows == that.rows && cols == that.cols && uuid.equals(that.uuid)
+ && (cache != null ? cache.equals(that.cache) : that.cache == null);
}
/**
*
*/
- private BlockEntry getEntryById(long blockId) {
- BlockMatrixKey key = getCacheKey(blockId);
+ private BlockEntry getEntryById(IgnitePair<Long> blockId) {
+ BlockMatrixKey key = getCacheKey(blockId.get1(), blockId.get2());
BlockEntry entry = cache.localPeek(key);
entry = entry != null ? entry : cache.get(key);
if (entry == null) {
- long colId = blockId == 0 ? 0 : blockId + 1;
- boolean isLastRow = (blockId) >= blocksInRow * (blocksInCol - 1);
- boolean isLastCol = (colId) % blocksInRow == 0;
+ int colSize = blockId.get1() == (blocksInCol - 1) ? rows % maxBlockEdge : maxBlockEdge;
+ int rowSize = blockId.get2() == (blocksInRow -1 ) ? cols % maxBlockEdge : maxBlockEdge;
- entry = new BlockEntry(isLastRow && rows % maxBlockEdge != 0 ? rows % maxBlockEdge : maxBlockEdge, isLastCol && cols % maxBlockEdge != 0 ? cols % maxBlockEdge : maxBlockEdge);
+ entry = new BlockEntry(rowSize, colSize);
}
return entry;
}
/**
- *
- */
- private long numberOfBlocks() {
- int rows = rowSize();
- int cols = columnSize();
-
- return ((rows / maxBlockEdge) + (((rows % maxBlockEdge) > 0) ? 1 : 0))
- * ((cols / maxBlockEdge) + (((cols % maxBlockEdge) > 0) ? 1 : 0));
- }
-
- /**
* TODO: IGNITE-5646, WIP
*
* Get affinity key for the given id.
*/
- private IgniteUuid getAffinityKey(long id) {
+ private IgniteUuid getAffinityKey(long blockIdRow, long blockIdCol) {
return null;
}
@@ -356,12 +334,12 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
* @param v New value to set.
*/
private void matrixSet(int a, int b, double v) {
- long id = getBlockId(a, b);
+ IgnitePair<Long> blockId = getBlockId(a, b);
// Remote set on the primary node (where given row or column is stored locally).
- ignite().compute(groupForKey(CACHE_NAME, id)).run(() -> {
+ ignite().compute(groupForKey(CACHE_NAME, blockId)).run(() -> {
IgniteCache<BlockMatrixKey, BlockEntry> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);
- BlockMatrixKey key = getCacheKey(getBlockId(a, b));
+ BlockMatrixKey key = getCacheKey(blockId.get1(), blockId.get2());
// Local get.
BlockEntry block = cache.localPeek(key, CachePeekMode.PRIMARY);
@@ -380,21 +358,26 @@ public class BlockMatrixStorage extends CacheUtils implements MatrixStorage, Sto
}
/** */
- private long getBlockId(int x, int y) {
- return (y / maxBlockEdge) * blockShift(cols) + (x / maxBlockEdge);
+ public IgnitePair<Long> getBlockId(int x, int y) {
+ return new IgnitePair<>((long)x / maxBlockEdge, (long)y / maxBlockEdge);
}
/** */
private BlockEntry initBlockFor(int x, int y) {
- int blockRows = rows - x >= maxBlockEdge ? maxBlockEdge : rows - x;
- int blockCols = cols - y >= maxBlockEdge ? maxBlockEdge : cols - y;
+ int blockRows = 0;
+ int blockCols = 0;
- return new BlockEntry(blockRows, blockCols);
- }
+ if(rows >= maxBlockEdge)
+ blockRows = rows - x >= maxBlockEdge ? maxBlockEdge : rows - x;
+ else
+ blockRows = rows;
- /** */
- private int blockShift(int i) {
- return (i) / maxBlockEdge + ((i) % maxBlockEdge > 0 ? 1 : 0);
+ if(cols >= maxBlockEdge)
+ blockCols = cols - y >= maxBlockEdge ? maxBlockEdge : cols - y;
+ else
+ blockCols = cols;
+
+ return new BlockEntry(blockRows, blockCols);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/94dd19d3/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
index 56ff638..35329ae 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/impls/matrix/SparseDistributedBlockMatrixTest.java
@@ -23,12 +23,10 @@ import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.Collection;
-import java.util.HashSet;
import java.util.Set;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.ml.math.Matrix;
import org.apache.ignite.ml.math.distributed.DistributedStorage;
import org.apache.ignite.ml.math.distributed.keys.impl.BlockMatrixKey;
@@ -293,6 +291,8 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest {
Matrix res = cacheMatrix1.times(cacheMatrix2);
+ BlockMatrixStorage storage = (BlockMatrixStorage)res.getStorage();
+
for(int i = 0; i < size; i++)
for(int j = 0; j < size; j++)
if (i == j)
@@ -364,17 +364,8 @@ public class SparseDistributedBlockMatrixTest extends GridCommonAbstractTest {
/** Build key set for SparseBlockDistributedMatrix. */
private Set<BlockMatrixKey> buildKeySet(SparseBlockDistributedMatrix m){
- Set<BlockMatrixKey> set = new HashSet<>();
-
BlockMatrixStorage storage = (BlockMatrixStorage)m.getStorage();
- IgniteUuid uuid = storage.getUUID();
-
- long maxBlock = (rows / 32 + (rows % 32 > 0 ? 1 : 0)) * (cols / 32 + (cols % 32 > 0 ? 1 : 0));
-
- for (long i = 0; i < maxBlock; i++)
- set.add(new BlockMatrixKey(i,uuid,null));
-
- return set;
+ return storage.getAllKeys();
}
}
[19/19] ignite git commit: IGNITE-6778 Fixing tests
Posted by ag...@apache.org.
IGNITE-6778 Fixing tests
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/98ca614b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/98ca614b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/98ca614b
Branch: refs/heads/ignite-6778
Commit: 98ca614b47ed114f9041d1c39e0572833683cc47
Parents: 54ccc9f
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Wed Nov 1 12:55:14 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Wed Nov 1 12:55:14 2017 +0300
----------------------------------------------------------------------
.../internal/processors/cache/CacheGroupContext.java | 4 +++-
.../internal/processors/cache/GridCacheProcessor.java | 6 ++++--
.../internal/processors/query/GridQueryIndexing.java | 4 ++--
.../internal/processors/query/GridQueryProcessor.java | 12 ++++++------
.../cache/IgniteClientCacheInitializationFailTest.java | 2 +-
.../internal/processors/query/h2/IgniteH2Indexing.java | 9 ++-------
6 files changed, 18 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ca614b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 19ca9c8..d9523e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -342,7 +342,9 @@ public class CacheGroupContext {
public GridCacheContext singleCacheContext() {
List<GridCacheContext> caches = this.caches;
- assert !sharedGroup() && caches.size() == 1 : ctx.kernalContext().isStopping();
+ assert !sharedGroup() && caches.size() == 1 :
+ "stopping=" + ctx.kernalContext().isStopping() + ", groupName=" + ccfg.getGroupName() +
+ ", caches=" + caches;
return caches.get(0);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ca614b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 021807a..01d0d41 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1026,7 +1026,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
assert desc != null : cctx.name();
- ctx.query().onCacheStop0(cctx.name(), false);
+ boolean rmvIdx = !cache.context().group().persistenceEnabled();
+
+ ctx.query().onCacheStop0(cctx.name(), rmvIdx);
ctx.query().onCacheStart0(cctx, desc.schema());
}
}
@@ -1150,7 +1152,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
cache.stop();
- ctx.kernalContext().query().onCacheStop(ctx, destroy);
+ ctx.kernalContext().query().onCacheStop(ctx, !cache.context().group().persistenceEnabled() || destroy);
if (isNearEnabled(ctx)) {
GridDhtCacheAdapter dht = ctx.near().dht();
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ca614b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index b0a3831..7a5cbc7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -195,10 +195,10 @@ public interface GridQueryIndexing {
* Unregisters cache.
*
* @param cacheName Cache name.
- * @param destroy Destroy flag.
+ * @param rmvIdx If {@code true}, will remove index.
* @throws IgniteCheckedException If failed to drop cache schema.
*/
- public void unregisterCache(String cacheName, boolean destroy) throws IgniteCheckedException;
+ public void unregisterCache(String cacheName, boolean rmvIdx) throws IgniteCheckedException;
/**
* Registers type if it was not known before or updates it otherwise.
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ca614b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 9327cfe..aa947ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -846,9 +846,9 @@ public class GridQueryProcessor extends GridProcessorAdapter {
/**
* @param cctx Cache context.
- * @param destroy Destroy flag.
+ * @param removeIdx If {@code true}, will remove index.
*/
- public void onCacheStop(GridCacheContext cctx, boolean destroy) {
+ public void onCacheStop(GridCacheContext cctx, boolean removeIdx) {
if (idx == null)
return;
@@ -856,7 +856,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
return;
try {
- onCacheStop0(cctx.name(), destroy);
+ onCacheStop0(cctx.name(), removeIdx);
}
finally {
busyLock.leaveBusy();
@@ -1527,9 +1527,9 @@ public class GridQueryProcessor extends GridProcessorAdapter {
* Use with {@link #busyLock} where appropriate.
*
* @param cacheName Cache name.
- * @param destroy Destroy flag.
+ * @param rmvIdx If {@code true}, will remove index.
*/
- public void onCacheStop0(String cacheName, boolean destroy) {
+ public void onCacheStop0(String cacheName, boolean rmvIdx) {
if (idx == null)
return;
@@ -1567,7 +1567,7 @@ public class GridQueryProcessor extends GridProcessorAdapter {
// Notify indexing.
try {
- idx.unregisterCache(cacheName, destroy);
+ idx.unregisterCache(cacheName, rmvIdx);
}
catch (Exception e) {
U.error(log, "Failed to clear indexing on cache unregister (will ignore): " + cacheName, e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ca614b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
index b0b758a..9126e96 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -299,7 +299,7 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT
}
/** {@inheritDoc} */
- @Override public void unregisterCache(String spaceName, boolean destroy) throws IgniteCheckedException {
+ @Override public void unregisterCache(String spaceName, boolean rmvIdx) throws IgniteCheckedException {
// No-op
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/98ca614b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index dd7308a..4a9faa1 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -61,7 +61,6 @@ import org.apache.ignite.internal.GridTopic;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -2330,7 +2329,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/** {@inheritDoc} */
- @Override public void unregisterCache(String cacheName, boolean destroy) {
+ @Override public void unregisterCache(String cacheName, boolean rmvIdx) {
String schemaName = schema(cacheName);
H2Schema schema = schemas.get(schemaName);
@@ -2345,14 +2344,10 @@ public class IgniteH2Indexing implements GridQueryIndexing {
// Drop tables.
Collection<H2TableDescriptor> rmvTbls = new HashSet<>();
- CacheGroupContext grpCtx = ctx.cache().cache(cacheName).context().group();
-
for (H2TableDescriptor tbl : schema.tables()) {
if (F.eq(tbl.cache().name(), cacheName)) {
try {
- boolean removeIdx = !grpCtx.persistenceEnabled() || destroy;
-
- tbl.table().setRemoveIndexOnDestroy(removeIdx);
+ tbl.table().setRemoveIndexOnDestroy(rmvIdx);
dropTable(tbl);
}
[16/19] ignite git commit: ignite-6788 Ignite WAL reader fails on Tx
marker record for persistent store with new style folder naming
Posted by ag...@apache.org.
ignite-6788 Ignite WAL reader fails on Tx marker record for persistent store with new style folder naming
Signed-off-by: agura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5542c70d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5542c70d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5542c70d
Branch: refs/heads/ignite-6778
Commit: 5542c70da913ff1006973e25f5112152674880db
Parents: 67ca745
Author: dpavlov <dp...@gridgain.com>
Authored: Wed Nov 1 03:17:00 2017 +0300
Committer: agura <ag...@apache.org>
Committed: Wed Nov 1 03:17:00 2017 +0300
----------------------------------------------------------------------
.../wal/reader/StandaloneGridKernalContext.java | 20 +-
.../db/wal/reader/IgniteWalReaderTest.java | 283 ++++++++++++-------
2 files changed, 195 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index c0c3650..485458b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
import java.io.File;
-import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.Iterator;
import java.util.List;
@@ -49,8 +48,8 @@ import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
@@ -95,6 +94,9 @@ import org.jetbrains.annotations.Nullable;
* Dummy grid kernal context
*/
public class StandaloneGridKernalContext implements GridKernalContext {
+ /** Binary metadata file store folderŅ */
+ public static final String BINARY_META_FOLDER = "binary_meta";
+
/** Config for fake Ignite instance. */
private final IgniteConfiguration cfg;
@@ -123,13 +125,12 @@ public class StandaloneGridKernalContext implements GridKernalContext {
* Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects will be provided <br>
*/
StandaloneGridKernalContext(IgniteLogger log,
- @Nullable final File binaryMetadataFileStoreDir,
- @Nullable final File marshallerMappingFileStoreDir) throws IgniteCheckedException {
+ @Nullable File binaryMetadataFileStoreDir,
+ @Nullable File marshallerMappingFileStoreDir) throws IgniteCheckedException {
this.log = log;
try {
- pluginProc = new StandaloneIgnitePluginProcessor(
- this, config());
+ pluginProc = new StandaloneIgnitePluginProcessor(this, config());
}
catch (IgniteCheckedException e) {
throw new IllegalStateException("Must not fail on empty providers list.", e);
@@ -137,7 +138,12 @@ public class StandaloneGridKernalContext implements GridKernalContext {
this.marshallerCtx = new MarshallerContextImpl(null);
this.cfg = prepareIgniteConfiguration();
- this.cacheObjProcessor = binaryMetadataFileStoreDir != null ? binaryProcessor(this, binaryMetadataFileStoreDir) : null;
+
+ // Fake folder provided to perform processor startup on empty folder.
+ if (binaryMetadataFileStoreDir == null)
+ binaryMetadataFileStoreDir = new File(BINARY_META_FOLDER).getAbsoluteFile();
+
+ this.cacheObjProcessor = binaryProcessor(this, binaryMetadataFileStoreDir);
if (marshallerMappingFileStoreDir != null) {
marshallerCtx.setMarshallerMappingFileStoreDir(marshallerMappingFileStoreDir);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index 88eacef..9348a68 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -40,6 +41,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteEvents;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheRebalanceMode;
@@ -68,8 +70,12 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.reader.Ignite
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.logger.NullLogger;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.jetbrains.annotations.NotNull;
@@ -77,6 +83,7 @@ import org.jetbrains.annotations.Nullable;
import org.junit.Assert;
import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor.genNewStyleSubfolderName;
@@ -94,12 +101,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
/** additional cache for testing different combinations of types in WAL */
private static final String CACHE_ADDL_NAME = "cache1";
- /** Delete DB dir before test. */
- private static final boolean deleteBefore = true;
-
- /** Delete DB dir after test. */
- private static final boolean deleteAfter = true;
-
/** Dump records to logger. Should be false for non local run */
private static final boolean dumpRecords = false;
@@ -115,6 +116,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
/** Custom wal mode. */
private WALMode customWalMode;
+ /** Clear properties in afterTest method() */
+ private boolean clearProperties;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
final IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -151,23 +155,22 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
@Override protected void beforeTest() throws Exception {
stopAllGrids();
- if (deleteBefore)
- deleteWorkFiles();
+ deleteWorkFiles();
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
stopAllGrids();
- if (deleteAfter)
- deleteWorkFiles();
+ if (clearProperties)
+ System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
}
/**
* @throws IgniteCheckedException If failed.
*/
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+ private void deleteWorkFiles() throws Exception {
+ GridTestUtils.deleteDbFiles();
}
/**
@@ -197,17 +200,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final int cntUsingMockIter = iterateAndCount(it, false);
log.info("Total records loaded " + cntUsingMockIter);
- assert cntUsingMockIter > 0;
- assert cntUsingMockIter > cacheObjectsToWrite;
+ assertTrue(cntUsingMockIter > 0);
+ assertTrue(cntUsingMockIter > cacheObjectsToWrite);
final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
final File walWorkDirWithConsistentId = new File(wal, subfolderName);
-
- final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
- final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
- final File marshaller = U.resolveWorkDirectory(workDir, "marshaller", false);
-
- final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithConsId, marshaller);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
log.info("Total records loaded using directory : " + cntArchiveDir);
@@ -218,12 +216,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
- assert cntArchiveFileByFile > cacheObjectsToWrite;
- assert cntArchiveDir > cacheObjectsToWrite;
- assert cntArchiveDir == cntArchiveFileByFile;
+ assertTrue(cntArchiveFileByFile > cacheObjectsToWrite);
+ assertTrue(cntArchiveDir > cacheObjectsToWrite);
+ assertTrue(cntArchiveDir == cntArchiveFileByFile);
//really count2 may be less because work dir correct loading is not supported yet
- assert cntUsingMockIter >= cntArchiveDir
- : "Mock based reader loaded " + cntUsingMockIter + " records but standalone has loaded only " + cntArchiveDir;
+ assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " +
+ "but standalone has loaded only " + cntArchiveDir,
+ cntUsingMockIter >= cntArchiveDir);
final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
@@ -231,10 +230,10 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
log.info("Total records loaded from work: " + cntWork);
- assert cntWork + cntArchiveFileByFile == cntUsingMockIter
- : "Work iterator loaded [" + cntWork + "] " +
- "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
- "mock iterator [" + cntUsingMockIter + "]";
+ assertTrue("Work iterator loaded [" + cntWork + "] " +
+ "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
+ "mock iterator [" + cntUsingMockIter + "]",
+ cntWork + cntArchiveFileByFile == cntUsingMockIter);
}
/**
@@ -312,7 +311,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
putDummyRecords(ignite, 500);
stopGrid("node0");
- assert evtRecorded.get();
+ assertTrue(evtRecorded.get());
}
/**
@@ -407,7 +406,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
archiveSegmentForInactivity.await(archiveIncompleteSegmentAfterInactivityMs + 1001, TimeUnit.MILLISECONDS);
stopGrid("node0");
- assert recordedAfterSleep;
+ assertTrue(recordedAfterSleep);
}
/**
@@ -460,8 +459,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
binaryMetaWithConsId,
marshallerMapping);
- final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>() {
- @Override public void accept(Object key, Object val) {
+ final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {
+ @Override public void apply(Object key, Object val) {
boolean rmv = remove(ctrlMap, key, val);
if (!rmv)
log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]");
@@ -471,14 +470,14 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
assertEquals(indexedObj.iVal, indexedObj.jVal);
assertEquals(indexedObj.iVal, key);
for (byte datum : indexedObj.getData()) {
- assert datum >= 'A' && datum <= 'A' + 10;
+ assertTrue(datum >= 'A' && datum <= 'A' + 10);
}
}
}
};
scanIterateAndCount(factory, workDir, subfolderName, cntEntries, txCnt, objConsumer, null);
- assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries " + ctrlMap;
+ assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty());
}
/**
@@ -510,8 +509,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final String subfolderName,
final int minCntEntries,
final int minTxCnt,
- @Nullable final BiConsumer<Object, Object> objConsumer,
- @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException {
+ @Nullable final IgniteBiInClosure<Object, Object> objConsumer,
+ @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
final File wal = new File(db, "wal");
@@ -544,8 +543,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final int entriesWork = valuesSum(cntWork.values());
log.info("Archive directory: Tx found " + txCntObservedWork + " entries " + entriesWork);
- assert entriesArch + entriesWork >= minCntEntries;
- assert txCntObservedWork + txCntObservedArch >= minTxCnt;
+ assertTrue("entriesArch=" + entriesArch + " + entriesWork=" + entriesWork
+ + " >= minCntEntries=" + minCntEntries,
+ entriesArch + entriesWork >= minCntEntries);
+ assertTrue("txCntObservedWork=" + txCntObservedWork + " + txCntObservedArch=" + txCntObservedArch
+ + " >= minTxCnt=" + minTxCnt,
+ txCntObservedWork + txCntObservedArch >= minTxCnt);
}
/**
@@ -611,11 +614,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final File binaryMetaWithNodeSubfolder = new File(binaryMeta, subfolderName);
final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
- final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
- binaryMetaWithNodeSubfolder,
- marshallerMapping);
- final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>() {
- @Override public void accept(Object key, Object val) {
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+ final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {
+ @Override public void apply(Object key, Object val) {
log.info("K: [" + key + ", " +
(key != null ? key.getClass().getName() : "?") + "]" +
" V: [" + val + ", " +
@@ -625,12 +626,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
String msg = "Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]";
log.error(msg);
}
- assert !(val instanceof BinaryObject);
+ assertFalse(val instanceof BinaryObject);
}
};
- final Consumer<DataRecord> toStrChecker = new Consumer<DataRecord>() {
- @Override public void accept(DataRecord record) {
+ final IgniteInClosure<DataRecord> toStrChecker = new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord record) {
String strRepresentation = record.toString();
for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
final String next = iter.next();
@@ -643,17 +644,17 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
};
scanIterateAndCount(factory, workDir, subfolderName, cntEntries, 0, objConsumer, toStrChecker);
- assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries: " + ctrlMap;
- assert ctrlStringsToSearch.isEmpty() : " Control Map for strings in entries is not empty after" +
- " reading records: " + ctrlStringsToSearch;
+ assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
+ assertTrue(" Control Map for strings in entries is not empty after" +
+ " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
//Validate same WAL log with flag binary objects only
final IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
binaryMetaWithNodeSubfolder,
marshallerMapping,
true);
- final BiConsumer<Object, Object> binObjConsumer = new BiConsumer<Object, Object>() {
- @Override public void accept(Object key, Object val) {
+ final IgniteBiInClosure<Object, Object> binObjConsumer = new IgniteBiInClosure<Object, Object>() {
+ @Override public void apply(Object key, Object val) {
log.info("K(KeepBinary): [" + key + ", " +
(key != null ? key.getClass().getName() : "?") + "]" +
" V(KeepBinary): [" + val + ", " +
@@ -693,15 +694,15 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
byte data[] = binaryObj.field("data");
for (byte datum : data) {
- assert datum >= 'A' && datum <= 'A' + 10;
+ assertTrue(datum >= 'A' && datum <= 'A' + 10);
}
}
}
}
};
- final Consumer<DataRecord> binObjToStrChecker = new Consumer<DataRecord>() {
- @Override public void accept(DataRecord record) {
+ final IgniteInClosure<DataRecord> binObjToStrChecker = new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord record) {
String strRepresentation = record.toString();
for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
final String next = iter.next();
@@ -714,9 +715,11 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
};
scanIterateAndCount(keepBinFactory, workDir, subfolderName, cntEntries, 0, binObjConsumer, binObjToStrChecker);
- assert ctrlMapForBinaryObjects.isEmpty() : " Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects;
- assert ctrlStringsForBinaryObjSearch.isEmpty() : " Control Map for strings in entries is not empty after" +
- " reading records: " + ctrlStringsForBinaryObjSearch;
+ assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects,
+ ctrlMapForBinaryObjects.isEmpty());
+ assertTrue(" Control Map for strings in entries is not empty after" +
+ " reading records: " + ctrlStringsForBinaryObjSearch,
+ ctrlStringsForBinaryObjSearch.isEmpty());
}
@@ -764,7 +767,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
stopGrid("node0");
final String workDir = U.defaultWorkDirectory();
- final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
scanIterateAndCount(factory, workDir, subfolderName, totalEntries, 0, null, null);
}
@@ -787,7 +790,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
stopGrid("node0");
final String workDir = U.defaultWorkDirectory();
- final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, null);
}
@@ -859,13 +862,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
stopGrid("node0");
final String workDir = U.defaultWorkDirectory();
- final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
final StringBuilder builder = new StringBuilder();
final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
- scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new Consumer<DataRecord>() {
- @Override public void accept(DataRecord dataRecord) {
+ scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord dataRecord) {
final List<DataEntry> entries = dataRecord.writeEntries();
builder.append("{");
@@ -902,14 +905,121 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
}
/**
- * @param subfolderName Subfolder name.
+ * Tests transaction generation and WAL for putAll cache operation.
+ * @throws Exception if failed.
+ */
+ public void testPutAllTxIntoTwoNodes() throws Exception {
+ final Ignite ignite = startGrid("node0");
+ final Ignite ignite1 = startGrid(1);
+
+ ignite.active(true);
+
+ final Map<Object, IndexedObject> map = new TreeMap<>();
+
+ final int cntEntries = 1000;
+ for (int i = 0; i < cntEntries; i++)
+ map.put(i, new IndexedObject(i));
+
+ ignite.cache(CACHE_NAME).putAll(map);
+
+ ignite.active(false);
+
+ final String subfolderName = genDbSubfolderName(ignite, 0);
+ final String subfolderName1 = genDbSubfolderName(ignite1, 1);
+
+ stopAllGrids();
+
+ final String workDir = U.defaultWorkDirectory();
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+
+ final StringBuilder builder = new StringBuilder();
+ final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
+
+ final IgniteInClosure<DataRecord> drHnd = new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord dataRecord) {
+ final List<DataEntry> entries = dataRecord.writeEntries();
+
+ builder.append("{");
+ for (DataEntry entry : entries) {
+ final GridCacheOperation op = entry.op();
+ final Integer cnt = operationsFound.get(op);
+
+ operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
+
+ if (entry instanceof UnwrapDataEntry) {
+ final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
+
+ builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
+ final GridCacheVersion ver = entry.nearXidVersion();
+
+ builder.append(", ");
+
+ if (ver != null)
+ builder.append("tx=").append(ver).append(", ");
+ }
+ }
+
+ builder.append("}\n");
+ }
+ };
+ scanIterateAndCount(factory, workDir, subfolderName, 1, 1, null, drHnd);
+ scanIterateAndCount(factory, workDir, subfolderName1, 1, 1, null, drHnd);
+
+ final Integer createsFound = operationsFound.get(CREATE);
+
+ if (log.isInfoEnabled())
+ log.info(builder.toString());
+
+ assertTrue("Create operations should be found in log: " + operationsFound,
+ createsFound != null && createsFound > 0);
+
+ assertTrue("Create operations count should be at least " + cntEntries + " in log: " + operationsFound,
+ createsFound != null && createsFound >= cntEntries);
+
+ }
+
+ /**
+ * Tests transaction generation and WAL for putAll cache operation.
+ * @throws Exception if failed.
+ */
+ public void testTxRecordsReadWoBinaryMeta() throws Exception {
+ clearProperties = true;
+ System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
+
+ final Ignite ignite = startGrid("node0");
+ ignite.active(true);
+
+ final Map<Object, IndexedObject> map = new TreeMap<>();
+
+ for (int i = 0; i < 1000; i++)
+ map.put(i, new IndexedObject(i));
+
+ ignite.cache(CACHE_NAME).putAll(map);
+
+ ignite.active(false);
+
+ final String workDir = U.defaultWorkDirectory();
+ final String subfolderName = genDbSubfolderName(ignite, 0);
+ stopAllGrids();
+
+ IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
+ PAGE_SIZE,
+ null,
+ null,
+ false);
+
+ scanIterateAndCount(factory, workDir, subfolderName, 1000, 1, null, null);
+ }
+
+ /**
* @param workDir Work directory.
+ * @param subfolderName Subfolder name.
* @return WAL iterator factory.
* @throws IgniteCheckedException If failed.
*/
@NotNull private IgniteWalIteratorFactory createWalIteratorFactory(
- String subfolderName,
- String workDir
+ final String workDir,
+ final String subfolderName
) throws IgniteCheckedException {
final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
@@ -918,7 +1028,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
return new IgniteWalIteratorFactory(log,
PAGE_SIZE,
binaryMetaWithConsId,
- marshallerMapping);
+ marshallerMapping,
+ false);
}
/**
@@ -943,8 +1054,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
*/
private Map<GridCacheVersion, Integer> iterateAndCountDataRecord(
final WALIterator walIter,
- @Nullable final BiConsumer<Object, Object> cacheObjHnd,
- @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException {
+ @Nullable final IgniteBiInClosure<Object, Object> cacheObjHnd,
+ @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
final Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
@@ -957,7 +1068,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final DataRecord dataRecord = (DataRecord)walRecord;
if (dataRecordHnd != null)
- dataRecordHnd.accept(dataRecord);
+ dataRecordHnd.apply(dataRecord);
final List<DataEntry> entries = dataRecord.writeEntries();
for (DataEntry entry : entries) {
@@ -990,7 +1101,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
"; Value: " + unwrappedValObj);
if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
- cacheObjHnd.accept(unwrappedKeyObj, unwrappedValObj);
+ cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 : entriesUnderTx + 1);
@@ -1008,36 +1119,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
return entriesUnderTxFound;
}
- /**
- * Represents an operation that accepts a single input argument and returns no
- * result.
- *
- * @param <T>
- */
- private interface Consumer<T> {
- /**
- * Performs this operation on the given argument.
- *
- * @param t the input argument
- */
- public void accept(T t);
- }
-
- /**
- * Represents an operation that accepts two input arguments and returns no
- * result.
- *
- * @param <T>
- */
- private interface BiConsumer<T, U> {
- /**
- * Performs this operation on the given argument.
- *
- * @param t the input argument
- */
- public void accept(T t, U u);
- }
-
/** Enum for cover binaryObject enum save/load */
enum TestEnum {
/** */A, /** */B, /** */C