You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2019/04/10 19:21:40 UTC
[ignite] branch ignite-9720 updated (55a6b2e -> 5ef026f)
This is an automated email from the ASF dual-hosted git repository.
sboikov pushed a change to branch ignite-9720
in repository https://gitbox.apache.org/repos/asf/ignite.git.
from 55a6b2e Merge remote-tracking branch 'origin/ignite-9720' into ignite-9720
add db38352 IGNITE-11654: [ML] Memory leak in KNNClassificationModel (#6392)
add 003c41e IGNITE-11647: [ML] ML Vectors should work with all Serializable objects besides double
add 3aa9167 IGNITE-11416 Added DistributedMetaStorage improvements - Fixes #6190.
new c8cec92 Merge remote-tracking branch 'remotes/origin/master' into ignite-9720
new 5ef026f fix tests
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/ignite/internal/GridComponent.java | 9 +-
.../org/apache/ignite/internal/IgniteFeatures.java | 6 +-
.../preloader/GridDhtPartitionsExchangeFuture.java | 13 +-
.../cache/persistence/GridCacheOffheapManager.java | 25 +-
.../cache/persistence/LazyCacheFreeList.java | 28 +-
.../cluster/GridClusterStateProcessor.java | 2 -
.../ReadableDistributedMetaStorage.java | 47 +-
.../persistence/DistributedMetaStorageBridge.java | 8 +-
.../DistributedMetaStorageClusterNodeData.java | 26 +-
.../DistributedMetaStorageHistoryCache.java | 228 +++++++
.../DistributedMetaStorageHistoryItem.java | 64 +-
.../persistence/DistributedMetaStorageImpl.java | 753 ++++++++++++++-------
.../DistributedMetaStorageJoiningNodeData.java | 16 +-
...ava => DistributedMetaStorageKeyValuePair.java} | 20 +-
.../DistributedMetaStorageUpdateAckMessage.java | 5 -
.../persistence/DistributedMetaStorageUtil.java | 24 +-
.../persistence/DistributedMetaStorageVersion.java | 3 +-
.../EmptyDistributedMetaStorageBridge.java | 10 +-
...InMemoryCachedDistributedMetaStorageBridge.java | 27 +-
.../NotAvailableDistributedMetaStorageBridge.java | 8 +-
.../ReadOnlyDistributedMetaStorageBridge.java | 107 +--
.../WritableDistributedMetaStorageBridge.java | 35 +-
.../db/checkpoint/CheckpointFreeListTest.java | 20 +-
.../DistributedMetaStoragePersistentTest.java | 94 ++-
.../metastorage/DistributedMetaStorageTest.java | 111 ++-
.../DistributedMetaStorageHistoryCacheTest.java | 227 +++++++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
.../ignite/ml/clustering/kmeans/KMeansModel.java | 2 +-
.../extractor/impl/BinaryObjectVectorizer.java | 9 +-
.../java/org/apache/ignite/ml/inference/Model.java | 4 +-
.../ignite/ml/knn/NNClassificationModel.java | 2 +-
.../knn/classification/KNNClassificationModel.java | 15 +
.../apache/ignite/ml/math/StorageOpsMetrics.java | 15 +-
.../ml/math/primitives/matrix/AbstractMatrix.java | 10 -
.../ignite/ml/math/primitives/matrix/Matrix.java | 5 +
.../ml/math/primitives/matrix/MatrixStorage.java | 5 +
.../math/primitives/matrix/impl/SparseMatrix.java | 2 +-
.../matrix/storage/DenseMatrixStorage.java | 12 +-
.../matrix/storage/SparseMatrixStorage.java | 12 +-
.../matrix/storage/ViewMatrixStorage.java | 10 -
.../ml/math/primitives/vector/AbstractVector.java | 78 ++-
.../ignite/ml/math/primitives/vector/Vector.java | 59 +-
.../ml/math/primitives/vector/VectorStorage.java | 27 +-
.../ml/math/primitives/vector/VectorUtils.java | 5 +-
.../primitives/vector/impl/DelegatingVector.java | 34 +-
.../math/primitives/vector/impl/SparseVector.java | 11 +-
.../vector/storage/DenseVectorStorage.java | 164 ++++-
.../vector/storage/SparseVectorStorage.java | 107 ++-
.../vector/storage/VectorViewStorage.java | 25 +-
.../storage/VectorizedViewMatrixStorage.java | 28 +-
.../ignite/ml/nn/ReplicatedVectorMatrix.java | 28 +-
.../ml/svm/SVMLinearClassificationTrainer.java | 10 +-
.../ml/dataset/feature/ObjectHistogramTest.java | 4 +-
.../java/org/apache/ignite/ml/math/BlasTest.java | 9 +-
.../ignite/ml/math/MathImplLocalTestSuite.java | 4 +
.../primitives/matrix/MatrixArrayStorageTest.java | 7 -
.../primitives/matrix/MatrixAttributeTest.java | 9 +-
.../matrix/MatrixViewConstructorTest.java | 2 -
.../math/primitives/vector/AbstractVectorTest.java | 20 -
.../vector/SparseVectorConstructorTest.java | 25 +-
.../primitives/vector/VectorAttributesTest.java | 20 +-
.../vector/VectorImplementationsFixtures.java | 4 +-
.../vector/VectorImplementationsTest.java | 3 +-
.../vector/storage/AbstractStorageTest.java | 147 ++++
.../vector/storage/DenseVectorStorageTest.java} | 44 +-
.../vector/storage/SparseVectorStorageTest.java} | 31 +-
.../ignite/ml/xgboost/XGModelComposition.java | 6 +-
67 files changed, 2105 insertions(+), 827 deletions(-)
create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageHistoryCache.java
copy modules/core/src/main/java/org/apache/ignite/internal/processors/metastorage/persistence/{DistributedMetaStorageHistoryItem.java => DistributedMetaStorageKeyValuePair.java} (72%)
create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/metastorage/persistence/DistributedMetaStorageHistoryCacheTest.java
create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/math/primitives/vector/storage/AbstractStorageTest.java
copy modules/{core/src/main/java/org/apache/ignite/internal/processors/query/property/QueryFieldAccessor.java => ml/src/test/java/org/apache/ignite/ml/math/primitives/vector/storage/DenseVectorStorageTest.java} (50%)
copy modules/{clients/src/test/java/org/apache/ignite/internal/client/ClientTcpSslMultiThreadedSelfTest.java => ml/src/test/java/org/apache/ignite/ml/math/primitives/vector/storage/SparseVectorStorageTest.java} (57%)
[ignite] 02/02: fix tests
Posted by sb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sboikov pushed a commit to branch ignite-9720
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 5ef026f00d2b97d5c5cfc8ec9e0f1efd8b8077d8
Author: sboikov <sb...@apache.org>
AuthorDate: Wed Apr 3 16:26:55 2019 +0300
fix tests
---
.../cache/persistence/GridCacheOffheapManager.java | 25 ++++++++++++++++---
.../cache/persistence/LazyCacheFreeList.java | 28 ++++++++++++----------
.../db/checkpoint/CheckpointFreeListTest.java | 20 ++++++++++++++--
3 files changed, 56 insertions(+), 17 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 4db3a86..1c09f02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -1550,10 +1550,29 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
RootPage reuseRoot = metas.reuseListRoot;
freeList = new LazyCacheFreeList() {
- @Override protected CacheFreeList<CacheDataRow> createDelegate() throws IgniteCheckedException {
- assert grp.shared().database().checkpointLockIsHeldByThread();
+ @Override protected CacheFreeList<CacheDataRow> createDelegate(boolean create) throws IgniteCheckedException {
+ boolean lock = false;
+
+ try {
+ if (!create) {
+ if (reuseRoot.pageId().pageId() == 0L)
+ return null;
+
+ assert !grp.shared().database().checkpointLockIsHeldByThread();
- return new PartitionCacheFreeList(grp, partId, reuseRoot);
+ grp.shared().database().checkpointReadLock();
+
+ lock = true;
+ }
+
+ assert grp.shared().database().checkpointLockIsHeldByThread();
+
+ return new PartitionCacheFreeList(grp, partId, reuseRoot);
+ }
+ finally {
+ if (lock)
+ grp.shared().database().checkpointReadUnlock();
+ }
}
};
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/LazyCacheFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/LazyCacheFreeList.java
index a337a26..9cdb573 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/LazyCacheFreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/LazyCacheFreeList.java
@@ -55,28 +55,30 @@ public abstract class LazyCacheFreeList implements CacheFreeList<CacheDataRow> {
/** {@inheritDoc} */
@Override public void insertDataRow(CacheDataRow row, IoStatisticsHolder statHolder) throws IgniteCheckedException {
- initDelegateIfNeeded().insertDataRow(row, statHolder);
+ initDelegateIfNeeded(true).insertDataRow(row, statHolder);
}
/** {@inheritDoc} */
@Override public boolean updateDataRow(long link, CacheDataRow row, IoStatisticsHolder statHolder) throws IgniteCheckedException {
- return initDelegateIfNeeded().updateDataRow(link, row, statHolder);
+ return initDelegateIfNeeded(true).updateDataRow(link, row, statHolder);
}
/** {@inheritDoc} */
@Override public <S, R> R updateDataRow(long link, PageHandler<S, R> pageHnd, S arg, IoStatisticsHolder statHolder) throws IgniteCheckedException {
- return (R)initDelegateIfNeeded().updateDataRow(link, pageHnd, arg, statHolder);
+ return (R)initDelegateIfNeeded(true).updateDataRow(link, pageHnd, arg, statHolder);
}
/** {@inheritDoc} */
@Override public void removeDataRowByLink(long link, IoStatisticsHolder statHolder) throws IgniteCheckedException {
- initDelegateIfNeeded().removeDataRowByLink(link, statHolder);
+ initDelegateIfNeeded(true).removeDataRowByLink(link, statHolder);
}
/** {@inheritDoc} */
@Override public int emptyDataPages() {
try {
- return initDelegateIfNeeded().emptyDataPages();
+ CacheFreeList<CacheDataRow> freeList = initDelegateIfNeeded(false);
+
+ return freeList != null ? freeList.emptyDataPages() : 0;
}
catch (IgniteCheckedException e) {
throw new IgniteException("Failed to initialize FreeList", e);
@@ -86,7 +88,9 @@ public abstract class LazyCacheFreeList implements CacheFreeList<CacheDataRow> {
/** {@inheritDoc} */
@Override public long freeSpace() {
try {
- return initDelegateIfNeeded().freeSpace();
+ CacheFreeList<CacheDataRow> freeList = initDelegateIfNeeded(false);
+
+ return freeList != null ? freeList.freeSpace() : null;
}
catch (IgniteCheckedException e) {
throw new IgniteException("Failed to initialize FreeList", e);
@@ -103,30 +107,30 @@ public abstract class LazyCacheFreeList implements CacheFreeList<CacheDataRow> {
/** {@inheritDoc} */
@Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
- initDelegateIfNeeded().addForRecycle(bag);
+ initDelegateIfNeeded(true).addForRecycle(bag);
}
/** {@inheritDoc} */
@Override public long takeRecycledPage() throws IgniteCheckedException {
- return initDelegateIfNeeded().takeRecycledPage();
+ return initDelegateIfNeeded(true).takeRecycledPage();
}
/** {@inheritDoc} */
@Override public long recycledPagesCount() throws IgniteCheckedException {
- return initDelegateIfNeeded().recycledPagesCount();
+ return initDelegateIfNeeded(true).recycledPagesCount();
}
/**
* @return Cache free list.
* @throws IgniteCheckedException If failed.
*/
- protected abstract CacheFreeList<CacheDataRow> createDelegate() throws IgniteCheckedException;
+ protected abstract CacheFreeList<CacheDataRow> createDelegate(boolean create) throws IgniteCheckedException;
/**
* @return Cache free list.
* @throws IgniteCheckedException If failed to initialize free list.
*/
- private CacheFreeList<CacheDataRow> initDelegateIfNeeded() throws IgniteCheckedException {
+ private CacheFreeList<CacheDataRow> initDelegateIfNeeded(boolean create) throws IgniteCheckedException {
CacheFreeList<CacheDataRow> delegate = this.delegate;
if (delegate != null)
@@ -141,7 +145,7 @@ public abstract class LazyCacheFreeList implements CacheFreeList<CacheDataRow> {
if (initLatchUpdater.compareAndSet(this, null, initLatch)) {
try {
- this.delegate = createDelegate();
+ this.delegate = createDelegate(create);
}
catch (IgniteCheckedException e) {
initErr = e;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointFreeListTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointFreeListTest.java
index 26441e5..0d080ce 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointFreeListTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/CheckpointFreeListTest.java
@@ -49,7 +49,10 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.persistence.LazyCacheFreeList;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -141,6 +144,19 @@ public class CheckpointFreeListTest extends GridCommonAbstractTest {
return cfg;
}
+ private AtomicReferenceArray<PagesList.Stripe[]> freeListBuckets(IgniteCacheOffheapManager.CacheDataStore cacheData) {
+ FreeList freeList = cacheData.rowStore().freeList();
+
+ if (freeList instanceof LazyCacheFreeList) {
+ freeList = GridTestUtils.getFieldValue(freeList, LazyCacheFreeList.class, "delegate");
+
+ if (freeList == null)
+ return null;
+ }
+
+ return U.field(freeList, "buckets");
+ }
+
/**
* @throws Exception if fail.
*/
@@ -168,7 +184,7 @@ public class CheckpointFreeListTest extends GridCommonAbstractTest {
HashMap<Integer, AtomicReferenceArray<PagesList.Stripe[]>> bucketsStorage = new HashMap<>();
offheap.cacheDataStores().forEach(cacheData ->
- bucketsStorage.put(cacheData.partId(), U.field(cacheData.rowStore().freeList(), "buckets"))
+ bucketsStorage.put(cacheData.partId(), freeListBuckets(cacheData))
);
forceCheckpoint();
@@ -182,7 +198,7 @@ public class CheckpointFreeListTest extends GridCommonAbstractTest {
GridCacheOffheapManager offheap2 = cacheOffheapManager();
offheap2.cacheDataStores().forEach(cacheData -> {
- AtomicReferenceArray<PagesList.Stripe[]> restoredBuckets = U.field(cacheData.rowStore().freeList(), "buckets");
+ AtomicReferenceArray<PagesList.Stripe[]> restoredBuckets = freeListBuckets(cacheData);
AtomicReferenceArray<PagesList.Stripe[]> savedBuckets = bucketsStorage.get(cacheData.partId());
if (savedBuckets != null && restoredBuckets != null) {
[ignite] 01/02: Merge remote-tracking branch
'remotes/origin/master' into ignite-9720
Posted by sb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sboikov pushed a commit to branch ignite-9720
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit c8cec928384f29a5db12a5519de251ebada44844
Merge: 55a6b2e 3aa9167
Author: sboikov <sb...@apache.org>
AuthorDate: Wed Apr 3 15:55:09 2019 +0300
Merge remote-tracking branch 'remotes/origin/master' into ignite-9720
.../org/apache/ignite/internal/GridComponent.java | 9 +-
.../org/apache/ignite/internal/IgniteFeatures.java | 6 +-
.../preloader/GridDhtPartitionsExchangeFuture.java | 13 +-
.../cluster/GridClusterStateProcessor.java | 2 -
.../ReadableDistributedMetaStorage.java | 47 +-
.../persistence/DistributedMetaStorageBridge.java | 8 +-
.../DistributedMetaStorageClusterNodeData.java | 26 +-
.../DistributedMetaStorageHistoryCache.java | 228 +++++++
.../DistributedMetaStorageHistoryItem.java | 64 +-
.../persistence/DistributedMetaStorageImpl.java | 753 ++++++++++++++-------
.../DistributedMetaStorageJoiningNodeData.java | 16 +-
...ava => DistributedMetaStorageKeyValuePair.java} | 20 +-
.../DistributedMetaStorageUpdateAckMessage.java | 5 -
.../persistence/DistributedMetaStorageUtil.java | 24 +-
.../persistence/DistributedMetaStorageVersion.java | 3 +-
.../EmptyDistributedMetaStorageBridge.java | 10 +-
...InMemoryCachedDistributedMetaStorageBridge.java | 27 +-
.../NotAvailableDistributedMetaStorageBridge.java | 8 +-
.../ReadOnlyDistributedMetaStorageBridge.java | 107 +--
.../WritableDistributedMetaStorageBridge.java | 35 +-
.../DistributedMetaStoragePersistentTest.java | 94 ++-
.../metastorage/DistributedMetaStorageTest.java | 111 ++-
.../DistributedMetaStorageHistoryCacheTest.java | 227 +++++++
.../ignite/testsuites/IgniteBasicTestSuite.java | 2 +
.../ignite/ml/clustering/kmeans/KMeansModel.java | 2 +-
.../extractor/impl/BinaryObjectVectorizer.java | 9 +-
.../java/org/apache/ignite/ml/inference/Model.java | 4 +-
.../ignite/ml/knn/NNClassificationModel.java | 2 +-
.../knn/classification/KNNClassificationModel.java | 15 +
.../apache/ignite/ml/math/StorageOpsMetrics.java | 15 +-
.../ml/math/primitives/matrix/AbstractMatrix.java | 10 -
.../ignite/ml/math/primitives/matrix/Matrix.java | 5 +
.../ml/math/primitives/matrix/MatrixStorage.java | 5 +
.../math/primitives/matrix/impl/SparseMatrix.java | 2 +-
.../matrix/storage/DenseMatrixStorage.java | 12 +-
.../matrix/storage/SparseMatrixStorage.java | 12 +-
.../matrix/storage/ViewMatrixStorage.java | 10 -
.../ml/math/primitives/vector/AbstractVector.java | 78 ++-
.../ignite/ml/math/primitives/vector/Vector.java | 59 +-
.../ml/math/primitives/vector/VectorStorage.java | 27 +-
.../ml/math/primitives/vector/VectorUtils.java | 5 +-
.../primitives/vector/impl/DelegatingVector.java | 34 +-
.../math/primitives/vector/impl/SparseVector.java | 11 +-
.../vector/storage/DenseVectorStorage.java | 164 ++++-
.../vector/storage/SparseVectorStorage.java | 107 ++-
.../vector/storage/VectorViewStorage.java | 25 +-
.../storage/VectorizedViewMatrixStorage.java | 28 +-
.../ignite/ml/nn/ReplicatedVectorMatrix.java | 28 +-
.../ml/svm/SVMLinearClassificationTrainer.java | 10 +-
.../ml/dataset/feature/ObjectHistogramTest.java | 4 +-
.../java/org/apache/ignite/ml/math/BlasTest.java | 9 +-
.../ignite/ml/math/MathImplLocalTestSuite.java | 4 +
.../primitives/matrix/MatrixArrayStorageTest.java | 7 -
.../primitives/matrix/MatrixAttributeTest.java | 9 +-
.../matrix/MatrixViewConstructorTest.java | 2 -
.../math/primitives/vector/AbstractVectorTest.java | 20 -
.../vector/SparseVectorConstructorTest.java | 25 +-
.../primitives/vector/VectorAttributesTest.java | 20 +-
.../vector/VectorImplementationsFixtures.java | 4 +-
.../vector/VectorImplementationsTest.java | 3 +-
.../vector/storage/AbstractStorageTest.java | 147 ++++
.../vector/storage/DenseVectorStorageTest.java | 61 ++
.../vector/storage/SparseVectorStorageTest.java} | 42 +-
.../ignite/ml/xgboost/XGModelComposition.java | 6 +-
64 files changed, 2101 insertions(+), 786 deletions(-)