You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by av...@apache.org on 2016/06/03 16:10:52 UTC
[1/3] ignite git commit: IGNITE-3228: Hadoop: workaround/fix for
inefficient memory usage.
Repository: ignite
Updated Branches:
refs/heads/ignite-3235 [created] 7b3f4944d
IGNITE-3228: Hadoop: workaround/fix for inefficient memory usage.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2490b0ae
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2490b0ae
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2490b0ae
Branch: refs/heads/ignite-3235
Commit: 2490b0aef17595247f8a6ae29482bc900e2a9a8d
Parents: 52a2637
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 2 09:11:09 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 2 09:11:09 2016 +0300
----------------------------------------------------------------------
.../shuffle/collections/HadoopMultimapBase.java | 90 +++++++++++++++++---
1 file changed, 76 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2490b0ae/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
index e6995ca..7dcff3d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
@@ -30,7 +30,6 @@ import org.apache.ignite.internal.processors.hadoop.HadoopTaskContext;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.HadoopDataInStream;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.HadoopDataOutStream;
import org.apache.ignite.internal.processors.hadoop.shuffle.streams.HadoopOffheapBuffer;
-import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory;
import org.jetbrains.annotations.Nullable;
@@ -48,7 +47,7 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
protected final int pageSize;
/** */
- private final Collection<GridLongList> allPages = new ConcurrentLinkedQueue<>();
+ private final Collection<Page> allPages = new ConcurrentLinkedQueue<>();
/**
* @param jobInfo Job info.
@@ -64,11 +63,12 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
}
/**
- * @param ptrs Page pointers.
+ * @param page Page.
*/
- private void deallocate(GridLongList ptrs) {
- while (!ptrs.isEmpty())
- mem.release(ptrs.remove(), ptrs.remove());
+ private void deallocate(Page page) {
+ assert page != null;
+
+ mem.release(page.ptr, page.size);
}
/**
@@ -105,8 +105,8 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
/** {@inheritDoc} */
@Override public void close() {
- for (GridLongList list : allPages)
- deallocate(list);
+ for (Page page : allPages)
+ deallocate(page);
}
/**
@@ -190,8 +190,8 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
/** */
private long writeStart;
- /** Size and pointer pairs list. */
- private final GridLongList pages = new GridLongList(16);
+ /** Current page. */
+ private Page curPage;
/**
* @param ctx Task context.
@@ -222,11 +222,10 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
private long allocateNextPage(long requestedSize) {
int writtenSize = writtenSize();
- long newPageSize = Math.max(writtenSize + requestedSize, pageSize);
+ long newPageSize = nextPageSize(writtenSize + requestedSize);
long newPagePtr = mem.allocate(newPageSize);
- pages.add(newPageSize);
- pages.add(newPagePtr);
+ System.out.println("ALLOCATED: " + newPageSize);
HadoopOffheapBuffer b = out.buffer();
@@ -240,10 +239,50 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
writeStart = newPagePtr;
+ // At this point old page is not needed, so we release it.
+ Page oldPage = curPage;
+
+ curPage = new Page(newPagePtr, newPageSize);
+
+ if (oldPage != null)
+ allPages.add(oldPage);
+
return b.move(requestedSize);
}
/**
+ * Get next page size.
+ *
+ * @param required Required amount of data.
+ * @return Next page size.
+ */
+ private long nextPageSize(long required) {
+ long pages = (required / pageSize) + 1;
+
+ long pagesPow2 = nextPowerOfTwo(pages);
+
+ return pagesPow2 * pageSize;
+ }
+
+ /**
+ * Get next power of two which greater or equal to the given number. Naive implementation.
+ *
+ * @param val Number
+ * @return Nearest pow2.
+ */
+ private long nextPowerOfTwo(long val) {
+ long res = 1;
+
+ while (res < val)
+ res = res << 1;
+
+ if (res < 0)
+ throw new IllegalArgumentException("Value is too big to find positive pow2: " + val);
+
+ return res;
+ }
+
+ /**
* @return Fixed pointer.
*/
private long fixAlignment() {
@@ -317,7 +356,8 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
/** {@inheritDoc} */
@Override public void close() throws IgniteCheckedException {
- allPages.add(pages);
+ if (curPage != null)
+ allPages.add(curPage);
keySer.close();
valSer.close();
@@ -372,4 +412,26 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
throw new UnsupportedOperationException();
}
}
+
+ /**
+ * Page.
+ */
+ private static class Page {
+ /** Pointer. */
+ private final long ptr;
+
+ /** Size. */
+ private final long size;
+
+ /**
+ * Constructor.
+ *
+ * @param ptr Pointer.
+ * @param size Size.
+ */
+ public Page(long ptr, long size) {
+ this.ptr = ptr;
+ this.size = size;
+ }
+ }
}
\ No newline at end of file
[3/3] ignite git commit: GNITE-3235 Failed to initialize primitive
boolean cache property of superclass
Posted by av...@apache.org.
GNITE-3235 Failed to initialize primitive boolean cache property of superclass
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7b3f4944
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7b3f4944
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7b3f4944
Branch: refs/heads/ignite-3235
Commit: 7b3f4944dd4feb92cf88ae13f4ad0818b1a4f159
Parents: 50a8ec2
Author: Anton Vinogradov <av...@apache.org>
Authored: Fri Jun 3 19:10:36 2016 +0300
Committer: Anton Vinogradov <av...@apache.org>
Committed: Fri Jun 3 19:10:36 2016 +0300
----------------------------------------------------------------------
.../internal/jdbc2/JdbcResultSetSelfTest.java | 60 +++++++++++++++++++-
.../processors/query/GridQueryProcessor.java | 58 +++++++++----------
2 files changed, 87 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7b3f4944/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcResultSetSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcResultSetSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcResultSetSelfTest.java
index 3607f53..8655b4f 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcResultSetSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcResultSetSelfTest.java
@@ -60,7 +60,8 @@ public class JdbcResultSetSelfTest extends GridCommonAbstractTest {
/** SQL query. */
private static final String SQL =
"select id, boolVal, byteVal, shortVal, intVal, longVal, floatVal, " +
- "doubleVal, bigVal, strVal, arrVal, dateVal, timeVal, tsVal, urlVal, f1, f2, f3, _val " +
+ "doubleVal, bigVal, strVal, arrVal, dateVal, timeVal, tsVal, urlVal, f1, f2, f3, _val, " +
+ "boolVal2, boolVal3 " +
"from TestObject where id = 1";
/** Statement. */
@@ -142,6 +143,8 @@ public class JdbcResultSetSelfTest extends GridCommonAbstractTest {
TestObject o = new TestObject(id);
o.boolVal = true;
+ o.boolVal2 = true;
+ o.boolVal3 = true;
o.byteVal = 1;
o.shortVal = 1;
o.intVal = 1;
@@ -182,6 +185,46 @@ public class JdbcResultSetSelfTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testBoolean2() throws Exception {
+ ResultSet rs = stmt.executeQuery(SQL);
+
+ int cnt = 0;
+
+ while (rs.next()) {
+ if (cnt == 0) {
+ assert rs.getBoolean("boolVal2");
+ assert rs.getBoolean(20);
+ }
+
+ cnt++;
+ }
+
+ assert cnt == 1;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testBoolean3() throws Exception {
+ ResultSet rs = stmt.executeQuery(SQL);
+
+ int cnt = 0;
+
+ while (rs.next()) {
+ if (cnt == 0) {
+ assert rs.getBoolean("boolVal3");
+ assert rs.getBoolean(21);
+ }
+
+ cnt++;
+ }
+
+ assert cnt == 1;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testByte() throws Exception {
ResultSet rs = stmt.executeQuery(SQL);
@@ -562,7 +605,20 @@ public class JdbcResultSetSelfTest extends GridCommonAbstractTest {
* Test object.
*/
@SuppressWarnings("UnusedDeclaration")
- private static class TestObject implements Serializable {
+ private static class BaseTestObject implements Serializable {
+ /** */
+ @QuerySqlField(index = false)
+ protected Boolean boolVal2;
+
+ @QuerySqlField(index = false)
+ protected boolean boolVal3;
+ }
+
+ /**
+ * Test object.
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ private static class TestObject extends BaseTestObject {
/** */
@QuerySqlField
private final int id;
http://git-wip-us.apache.org/repos/asf/ignite/blob/7b3f4944/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 074e7e5..442763f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -17,6 +17,30 @@
package org.apache.ignite.internal.processors.query;
+import java.lang.reflect.AccessibleObject;
+import java.lang.reflect.Field;
+import java.lang.reflect.Member;
+import java.lang.reflect.Method;
+import java.math.BigDecimal;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import javax.cache.Cache;
+import javax.cache.CacheException;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
@@ -66,31 +90,6 @@ import org.apache.ignite.spi.indexing.IndexingQueryFilter;
import org.jetbrains.annotations.Nullable;
import org.jsr166.ConcurrentHashMap8;
-import javax.cache.Cache;
-import javax.cache.CacheException;
-import java.lang.reflect.AccessibleObject;
-import java.lang.reflect.Field;
-import java.lang.reflect.Member;
-import java.lang.reflect.Method;
-import java.math.BigDecimal;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-
import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
import static org.apache.ignite.internal.IgniteComponentType.INDEXING;
import static org.apache.ignite.internal.processors.query.GridQueryIndexType.FULLTEXT;
@@ -1681,14 +1680,15 @@ public class GridQueryProcessor extends GridProcessorAdapter {
// No-op.
}
- if (tmp == null) {
+ Class cls0 = cls;
+
+ while (tmp == null && cls0 != null)
try {
- tmp = new ClassProperty(cls.getDeclaredField(prop), key, alias, coCtx);
+ tmp = new ClassProperty(cls0.getDeclaredField(prop), key, alias, coCtx);
}
catch (NoSuchFieldException ignored) {
- // No-op.
+ cls0 = cls0.getSuperclass();
}
- }
if (tmp == null) {
try {
[2/3] ignite git commit: IGNITE-3228: Minor fix.
Posted by av...@apache.org.
IGNITE-3228: Minor fix.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/50a8ec24
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/50a8ec24
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/50a8ec24
Branch: refs/heads/ignite-3235
Commit: 50a8ec24fcae6cb673a257789724dd5a2dbb734a
Parents: 2490b0a
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Thu Jun 2 11:47:38 2016 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Thu Jun 2 11:47:38 2016 +0300
----------------------------------------------------------------------
.../processors/hadoop/shuffle/collections/HadoopMultimapBase.java | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/50a8ec24/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
index 7dcff3d..39b7c51 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/collections/HadoopMultimapBase.java
@@ -225,8 +225,6 @@ public abstract class HadoopMultimapBase implements HadoopMultimap {
long newPageSize = nextPageSize(writtenSize + requestedSize);
long newPagePtr = mem.allocate(newPageSize);
- System.out.println("ALLOCATED: " + newPageSize);
-
HadoopOffheapBuffer b = out.buffer();
b.set(newPagePtr, newPageSize);