You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/12/20 08:04:09 UTC
[01/50] [abbrv] ignite git commit: IGNITE-6920 Fixed dependencies
resolving.
Repository: ignite
Updated Branches:
refs/heads/ignite-zk fe515ee55 -> d6ec00c0e
IGNITE-6920 Fixed dependencies resolving.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c10aa0c4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c10aa0c4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c10aa0c4
Branch: refs/heads/ignite-zk
Commit: c10aa0c4bfbd18ee03887c690c27defe8cdb4b3c
Parents: 9a8b95e
Author: Andrey Novikov <an...@gridgain.com>
Authored: Fri Nov 24 14:14:48 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Tue Dec 12 21:39:02 2017 +0700
----------------------------------------------------------------------
modules/web-console/backend/app/agentSocket.js | 8 ++---
.../web-console/backend/app/agentsHandler.js | 15 ++++----
modules/web-console/backend/app/apiServer.js | 9 +++--
.../web-console/backend/app/browsersHandler.js | 7 ++--
modules/web-console/backend/app/configure.js | 14 ++++++--
modules/web-console/backend/app/mongo.js | 10 +++---
modules/web-console/backend/app/mongoose.js | 8 ++---
modules/web-console/backend/app/nconf.js | 6 ++--
modules/web-console/backend/app/settings.js | 6 ++--
.../web-console/backend/routes/activities.js | 37 ++++++++++----------
modules/web-console/backend/routes/admin.js | 8 ++---
modules/web-console/backend/routes/caches.js | 6 ++--
modules/web-console/backend/routes/clusters.js | 6 ++--
.../web-console/backend/routes/configuration.js | 6 ++--
modules/web-console/backend/routes/demo.js | 7 ++--
modules/web-console/backend/routes/domains.js | 6 ++--
modules/web-console/backend/routes/downloads.js | 6 ++--
modules/web-console/backend/routes/igfss.js | 6 ++--
modules/web-console/backend/routes/notebooks.js | 7 ++--
modules/web-console/backend/routes/profile.js | 10 +++---
modules/web-console/backend/routes/public.js | 9 ++---
.../web-console/backend/services/activities.js | 7 ++--
modules/web-console/backend/services/auth.js | 5 ++-
modules/web-console/backend/services/caches.js | 7 ++--
.../web-console/backend/services/clusters.js | 7 ++--
.../backend/services/configurations.js | 5 ++-
modules/web-console/backend/services/domains.js | 7 ++--
.../web-console/backend/services/downloads.js | 13 +++----
modules/web-console/backend/services/igfss.js | 7 ++--
modules/web-console/backend/services/mails.js | 9 ++---
.../web-console/backend/services/notebooks.js | 7 ++--
.../backend/services/notifications.js | 5 ++-
.../web-console/backend/services/sessions.js | 5 ++-
modules/web-console/backend/services/users.js | 7 ++--
34 files changed, 167 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/agentSocket.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/agentSocket.js b/modules/web-console/backend/app/agentSocket.js
index 6e4518a..39d880f 100644
--- a/modules/web-console/backend/app/agentSocket.js
+++ b/modules/web-console/backend/app/agentSocket.js
@@ -17,14 +17,15 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
/**
* Module interaction with agents.
*/
module.exports = {
- implements: 'agent-socket',
- inject: ['require(lodash)']
+ implements: 'agent-socket'
};
/**
@@ -78,10 +79,9 @@ class Command {
}
/**
- * @param _
* @returns {AgentSocket}
*/
-module.exports.factory = function(_) {
+module.exports.factory = function() {
/**
* Connected agent descriptor.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/agentsHandler.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/agentsHandler.js b/modules/web-console/backend/app/agentsHandler.js
index 844ce1e..9ee64ce 100644
--- a/modules/web-console/backend/app/agentsHandler.js
+++ b/modules/web-console/backend/app/agentsHandler.js
@@ -19,6 +19,12 @@
const uuid = require('uuid/v4');
+const fs = require('fs');
+const path = require('path');
+const JSZip = require('jszip');
+const socketio = require('socket.io');
+const _ = require('lodash');
+
// Fire me up!
/**
@@ -26,21 +32,16 @@ const uuid = require('uuid/v4');
*/
module.exports = {
implements: 'agents-handler',
- inject: ['require(lodash)', 'require(fs)', 'require(path)', 'require(jszip)', 'require(socket.io)', 'settings', 'mongo', 'agent-socket']
+ inject: ['settings', 'mongo', 'agent-socket']
};
/**
- * @param _
- * @param fs
- * @param path
- * @param JSZip
- * @param socketio
* @param settings
* @param mongo
* @param {AgentSocket} AgentSocket
* @returns {AgentsHandler}
*/
-module.exports.factory = function(_, fs, path, JSZip, socketio, settings, mongo, AgentSocket) {
+module.exports.factory = function(settings, mongo, AgentSocket) {
class AgentSockets {
constructor() {
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/apiServer.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/apiServer.js b/modules/web-console/backend/app/apiServer.js
index cb09750..0030529 100644
--- a/modules/web-console/backend/app/apiServer.js
+++ b/modules/web-console/backend/app/apiServer.js
@@ -17,12 +17,17 @@
'use strict';
+const fs = require('fs');
+const path = require('path');
+
// Fire me up!
+const Express = require('express');
+
module.exports = {
implements: 'api-server',
- inject: ['require(fs)', 'require(path)', 'require(express)', 'settings', 'configure', 'routes'],
- factory(fs, path, Express, settings, configure, routes) {
+ inject: ['settings', 'configure', 'routes'],
+ factory(settings, configure, routes) {
/**
* Connected agents manager.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/browsersHandler.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index 7ae247b..198bc12 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -17,6 +17,9 @@
'use strict';
+const _ = require('lodash');
+const socketio = require('socket.io');
+
// Fire me up!
/**
@@ -24,8 +27,8 @@
*/
module.exports = {
implements: 'browsers-handler',
- inject: ['require(lodash)', 'require(socket.io)', 'configure', 'errors', 'mongo'],
- factory: (_, socketio, configure, errors, mongo) => {
+ inject: ['configure', 'errors', 'mongo'],
+ factory: (configure, errors, mongo) => {
class BrowserSockets {
constructor() {
this.sockets = new Map();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/configure.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/configure.js b/modules/web-console/backend/app/configure.js
index 7624bdd..b7bdb49 100644
--- a/modules/web-console/backend/app/configure.js
+++ b/modules/web-console/backend/app/configure.js
@@ -17,6 +17,15 @@
'use strict';
+const _ = require('lodash');
+const logger = require('morgan');
+const cookieParser = require('cookie-parser');
+const bodyParser = require('body-parser');
+const session = require('express-session');
+const connectMongo = require('connect-mongo');
+const passport = require('passport');
+const passportSocketIo = require('passport.socketio');
+
// Fire me up!
/**
@@ -24,11 +33,10 @@
*/
module.exports = {
implements: 'configure',
- inject: ['require(lodash)', 'require(morgan)', 'require(cookie-parser)', 'require(body-parser)',
- 'require(express-session)', 'require(connect-mongo)', 'require(passport)', 'require(passport.socketio)', 'settings', 'mongo', 'middlewares:*']
+ inject: ['settings', 'mongo', 'middlewares:*']
};
-module.exports.factory = function(_, logger, cookieParser, bodyParser, session, connectMongo, passport, passportSocketIo, settings, mongo, apis) {
+module.exports.factory = function(settings, mongo, apis) {
const _sessionStore = new (connectMongo(session))({mongooseConnection: mongo.connection});
return {
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/mongo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index e0d0a0f..de91676 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -17,6 +17,8 @@
'use strict';
+const passportMongo = require('passport-local-mongoose');
+
// Fire me up!
/**
@@ -24,10 +26,10 @@
*/
module.exports = {
implements: 'mongo',
- inject: ['require(passport-local-mongoose)', 'settings', 'mongoose']
+ inject: ['settings', 'mongoose']
};
-const defineSchema = (passportMongo, mongoose) => {
+const defineSchema = (mongoose) => {
const Schema = mongoose.Schema;
const ObjectId = mongoose.Schema.Types.ObjectId;
const result = { connection: mongoose.connection };
@@ -1150,7 +1152,7 @@ const defineSchema = (passportMongo, mongoose) => {
return result;
};
-module.exports.factory = function(passportMongo, settings, mongoose) {
+module.exports.factory = function(settings, mongoose) {
// Use native promises
mongoose.Promise = global.Promise;
@@ -1204,5 +1206,5 @@ module.exports.factory = function(passportMongo, settings, mongoose) {
return Promise.reject(err);
});
})
- .then(() => defineSchema(passportMongo, mongoose));
+ .then(() => defineSchema(mongoose));
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/mongoose.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/mongoose.js b/modules/web-console/backend/app/mongoose.js
index 7b6e7f3..94f9c60 100644
--- a/modules/web-console/backend/app/mongoose.js
+++ b/modules/web-console/backend/app/mongoose.js
@@ -17,13 +17,11 @@
'use strict';
+const mongoose = require('mongoose');
+
// Fire me up!
module.exports = {
implements: 'mongoose',
- inject: ['require(mongoose)']
-};
-
-module.exports.factory = (mongoose) => {
- return mongoose;
+ factory: () => mongoose
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/nconf.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/nconf.js b/modules/web-console/backend/app/nconf.js
index 6813f0f..3f5d7d1 100644
--- a/modules/web-console/backend/app/nconf.js
+++ b/modules/web-console/backend/app/nconf.js
@@ -17,6 +17,9 @@
'use strict';
+const fs = require('fs');
+const nconf = require('nconf');
+
// Fire me up!
/**
@@ -24,8 +27,7 @@
*/
module.exports = {
implements: 'nconf',
- inject: ['require(nconf)', 'require(fs)'],
- factory(nconf, fs) {
+ factory() {
nconf.env({separator: '_'}).argv();
const dfltFile = 'config/settings.json';
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/app/settings.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js
index 5032443..2d94eb5 100644
--- a/modules/web-console/backend/app/settings.js
+++ b/modules/web-console/backend/app/settings.js
@@ -17,6 +17,8 @@
'use strict';
+const fs = require('fs');
+
// Fire me up!
/**
@@ -24,8 +26,8 @@
*/
module.exports = {
implements: 'settings',
- inject: ['nconf', 'require(fs)'],
- factory(nconf, fs) {
+ inject: ['nconf'],
+ factory(nconf) {
/**
* Normalize a port into a number, string, or false.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/activities.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/activities.js b/modules/web-console/backend/routes/activities.js
index ad0e469..4db6ac2 100644
--- a/modules/web-console/backend/routes/activities.js
+++ b/modules/web-console/backend/routes/activities.js
@@ -17,29 +17,30 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/activities',
- inject: ['require(express)', 'services/activities']
-};
+ inject: ['services/activities'],
+ /**
+ * @param express
+ * @param {ActivitiesService} activitiesService
+ * @returns {Promise}
+ */
+ factory: (activitiesService) => {
+ return new Promise((factoryResolve) => {
+ const router = new express.Router();
-/**
- * @param express
- * @param {ActivitiesService} activitiesService
- * @returns {Promise}
- */
-module.exports.factory = function(express, activitiesService) {
- return new Promise((factoryResolve) => {
- const router = new express.Router();
+ // Post user activities to page.
+ router.post('/page', (req, res) => {
+ activitiesService.merge(req.user._id, req.body)
+ .then(res.api.ok)
+ .catch(res.api.error);
+ });
- // Post user activities to page.
- router.post('/page', (req, res) => {
- activitiesService.merge(req.user._id, req.body)
- .then(res.api.ok)
- .catch(res.api.error);
+ factoryResolve(router);
});
-
- factoryResolve(router);
- });
+ }
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/admin.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/admin.js b/modules/web-console/backend/routes/admin.js
index 5ee41c8..a9efc8d 100644
--- a/modules/web-console/backend/routes/admin.js
+++ b/modules/web-console/backend/routes/admin.js
@@ -17,16 +17,16 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/admin',
- inject: ['require(lodash)', 'require(express)', 'settings', 'mongo', 'services/spaces', 'services/mails', 'services/sessions', 'services/users', 'services/notifications']
+ inject: ['settings', 'mongo', 'services/spaces', 'services/mails', 'services/sessions', 'services/users', 'services/notifications']
};
/**
- * @param _
- * @param express
* @param settings
* @param mongo
* @param spacesService
@@ -36,7 +36,7 @@ module.exports = {
* @param {NotificationsService} notificationsService
* @returns {Promise}
*/
-module.exports.factory = function(_, express, settings, mongo, spacesService, mailsService, sessionsService, usersService, notificationsService) {
+module.exports.factory = function(settings, mongo, spacesService, mailsService, sessionsService, usersService, notificationsService) {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/caches.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/caches.js b/modules/web-console/backend/routes/caches.js
index e040fda..d7ed8b8 100644
--- a/modules/web-console/backend/routes/caches.js
+++ b/modules/web-console/backend/routes/caches.js
@@ -17,14 +17,16 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/caches',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/caches']
+ inject: ['mongo', 'services/caches']
};
-module.exports.factory = function(_, express, mongo, cachesService) {
+module.exports.factory = function(mongo, cachesService) {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/clusters.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/clusters.js b/modules/web-console/backend/routes/clusters.js
index 97a446a..24334c2 100644
--- a/modules/web-console/backend/routes/clusters.js
+++ b/modules/web-console/backend/routes/clusters.js
@@ -17,14 +17,16 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/clusters',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/clusters']
+ inject: ['mongo', 'services/clusters']
};
-module.exports.factory = function(_, express, mongo, clustersService) {
+module.exports.factory = function(mongo, clustersService) {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/configuration.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/configuration.js b/modules/web-console/backend/routes/configuration.js
index c3ff5d6..d9bde75 100644
--- a/modules/web-console/backend/routes/configuration.js
+++ b/modules/web-console/backend/routes/configuration.js
@@ -17,14 +17,16 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/configurations',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/configurations']
+ inject: ['mongo', 'services/configurations']
};
-module.exports.factory = function(_, express, mongo, configurationsService) {
+module.exports.factory = function(mongo, configurationsService) {
return new Promise((factoryResolve) => {
const router = new express.Router();
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/demo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/demo.js b/modules/web-console/backend/routes/demo.js
index b200d83..a18fa7a 100644
--- a/modules/web-console/backend/routes/demo.js
+++ b/modules/web-console/backend/routes/demo.js
@@ -17,6 +17,9 @@
'use strict';
+const express = require('express');
+const _ = require('lodash');
+
// Fire me up!
const clusters = require('./demo/clusters.json');
@@ -26,7 +29,7 @@ const igfss = require('./demo/igfss.json');
module.exports = {
implements: 'routes/demo',
- inject: ['require(lodash)', 'require(express)', 'errors', 'settings', 'mongo', 'services/spaces']
+ inject: ['errors', 'settings', 'mongo', 'services/spaces']
};
/**
@@ -39,7 +42,7 @@ module.exports = {
* @param spacesService
* @return {Promise}
*/
-module.exports.factory = (_, express, errors, settings, mongo, spacesService) => {
+module.exports.factory = (errors, settings, mongo, spacesService) => {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/domains.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/domains.js b/modules/web-console/backend/routes/domains.js
index db1d892..caa9201 100644
--- a/modules/web-console/backend/routes/domains.js
+++ b/modules/web-console/backend/routes/domains.js
@@ -17,14 +17,16 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/domains',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/domains']
+ inject: ['mongo', 'services/domains']
};
-module.exports.factory = (_, express, mongo, domainsService) => {
+module.exports.factory = (mongo, domainsService) => {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/downloads.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/downloads.js b/modules/web-console/backend/routes/downloads.js
index 88a1923..a06bb27 100644
--- a/modules/web-console/backend/routes/downloads.js
+++ b/modules/web-console/backend/routes/downloads.js
@@ -17,11 +17,13 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/downloads',
- inject: ['require(lodash)', 'require(express)', 'services/agents', 'services/activities']
+ inject: ['services/agents', 'services/activities']
};
/**
@@ -31,7 +33,7 @@ module.exports = {
* @param {ActivitiesService} activitiesService
* @returns {Promise}
*/
-module.exports.factory = function(_, express, downloadsService, activitiesService) {
+module.exports.factory = function(downloadsService, activitiesService) {
return new Promise((resolveFactory) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/igfss.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/igfss.js b/modules/web-console/backend/routes/igfss.js
index c88d627..b95f21f 100644
--- a/modules/web-console/backend/routes/igfss.js
+++ b/modules/web-console/backend/routes/igfss.js
@@ -17,14 +17,16 @@
'use strict';
+const express = require('express');
+
// Fire me up!
module.exports = {
implements: 'routes/igfss',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/igfss']
+ inject: ['mongo', 'services/igfss']
};
-module.exports.factory = function(_, express, mongo, igfssService) {
+module.exports.factory = function(mongo, igfssService) {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/notebooks.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/notebooks.js b/modules/web-console/backend/routes/notebooks.js
index c330809..0807db8 100644
--- a/modules/web-console/backend/routes/notebooks.js
+++ b/modules/web-console/backend/routes/notebooks.js
@@ -17,14 +17,17 @@
'use strict';
+const express = require('express');
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'routes/notebooks',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/spaces', 'services/notebooks']
+ inject: ['mongo', 'services/spaces', 'services/notebooks']
};
-module.exports.factory = (_, express, mongo, spacesService, notebooksService) => {
+module.exports.factory = (mongo, spacesService, notebooksService) => {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/profile.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/profile.js b/modules/web-console/backend/routes/profile.js
index 76edf72..0ce2656 100644
--- a/modules/web-console/backend/routes/profile.js
+++ b/modules/web-console/backend/routes/profile.js
@@ -17,22 +17,22 @@
'use strict';
+const express = require('express');
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'routes/profiles',
- inject: ['require(lodash)', 'require(express)', 'mongo', 'services/users']
+ inject: ['mongo', 'services/users']
};
/**
- *
- * @param _ Lodash module
- * @param express Express module
* @param mongo
* @param {UsersService} usersService
* @returns {Promise}
*/
-module.exports.factory = function(_, express, mongo, usersService) {
+module.exports.factory = function(mongo, usersService) {
return new Promise((resolveFactory) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/routes/public.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/routes/public.js b/modules/web-console/backend/routes/public.js
index 860e267..a643a00 100644
--- a/modules/web-console/backend/routes/public.js
+++ b/modules/web-console/backend/routes/public.js
@@ -17,23 +17,24 @@
'use strict';
+const express = require('express');
+const passport = require('passport');
+
// Fire me up!
module.exports = {
implements: 'routes/public',
- inject: ['require(express)', 'require(passport)', 'mongo', 'services/mails', 'services/users', 'services/auth']
+ inject: ['mongo', 'services/mails', 'services/users', 'services/auth']
};
/**
- * @param express
- * @param passport
* @param mongo
* @param mailsService
* @param {UsersService} usersService
* @param {AuthService} authService
* @returns {Promise}
*/
-module.exports.factory = function(express, passport, mongo, mailsService, usersService, authService) {
+module.exports.factory = function(mongo, mailsService, usersService, authService) {
return new Promise((factoryResolve) => {
const router = new express.Router();
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/activities.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/activities.js b/modules/web-console/backend/services/activities.js
index a049f65..9399eb0 100644
--- a/modules/web-console/backend/services/activities.js
+++ b/modules/web-console/backend/services/activities.js
@@ -17,19 +17,20 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/activities',
- inject: ['require(lodash)', 'mongo']
+ inject: ['mongo']
};
/**
- * @param _
* @param mongo
* @returns {ActivitiesService}
*/
-module.exports.factory = (_, mongo) => {
+module.exports.factory = (mongo) => {
class ActivitiesService {
/**
* Update page activities.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/auth.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/auth.js b/modules/web-console/backend/services/auth.js
index c3423da..dde2460 100644
--- a/modules/web-console/backend/services/auth.js
+++ b/modules/web-console/backend/services/auth.js
@@ -21,18 +21,17 @@
module.exports = {
implements: 'services/auth',
- inject: ['require(lodash)', 'mongo', 'settings', 'errors']
+ inject: ['mongo', 'settings', 'errors']
};
/**
- * @param _
* @param mongo
* @param settings
* @param errors
* @returns {AuthService}
*/
-module.exports.factory = (_, mongo, settings, errors) => {
+module.exports.factory = (mongo, settings, errors) => {
class AuthService {
/**
* Generate token string.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/caches.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/caches.js b/modules/web-console/backend/services/caches.js
index 19aed82..5c96ccd 100644
--- a/modules/web-console/backend/services/caches.js
+++ b/modules/web-console/backend/services/caches.js
@@ -17,21 +17,22 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/caches',
- inject: ['require(lodash)', 'mongo', 'services/spaces', 'errors']
+ inject: ['mongo', 'services/spaces', 'errors']
};
/**
- * @param _
* @param mongo
* @param {SpacesService} spaceService
* @param errors
* @returns {CachesService}
*/
-module.exports.factory = (_, mongo, spaceService, errors) => {
+module.exports.factory = (mongo, spaceService, errors) => {
/**
* Convert remove status operation to own presentation.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/clusters.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/clusters.js b/modules/web-console/backend/services/clusters.js
index 06e413a..9f50ede 100644
--- a/modules/web-console/backend/services/clusters.js
+++ b/modules/web-console/backend/services/clusters.js
@@ -17,21 +17,22 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/clusters',
- inject: ['require(lodash)', 'mongo', 'services/spaces', 'errors']
+ inject: ['mongo', 'services/spaces', 'errors']
};
/**
- * @param _
* @param mongo
* @param {SpacesService} spacesService
* @param errors
* @returns {ClustersService}
*/
-module.exports.factory = (_, mongo, spacesService, errors) => {
+module.exports.factory = (mongo, spacesService, errors) => {
/**
* Convert remove status operation to own presentation.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/configurations.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/configurations.js b/modules/web-console/backend/services/configurations.js
index 7eef8a2..36d9932 100644
--- a/modules/web-console/backend/services/configurations.js
+++ b/modules/web-console/backend/services/configurations.js
@@ -21,11 +21,10 @@
module.exports = {
implements: 'services/configurations',
- inject: ['require(lodash)', 'mongo', 'services/spaces', 'services/clusters', 'services/caches', 'services/domains', 'services/igfss']
+ inject: ['mongo', 'services/spaces', 'services/clusters', 'services/caches', 'services/domains', 'services/igfss']
};
/**
- * @param _
* @param mongo
* @param {SpacesService} spacesService
* @param {ClustersService} clustersService
@@ -34,7 +33,7 @@ module.exports = {
* @param {IgfssService} igfssService
* @returns {ConfigurationsService}
*/
-module.exports.factory = (_, mongo, spacesService, clustersService, cachesService, domainsService, igfssService) => {
+module.exports.factory = (mongo, spacesService, clustersService, cachesService, domainsService, igfssService) => {
class ConfigurationsService {
static list(userId, demo) {
let spaces;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/domains.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/domains.js b/modules/web-console/backend/services/domains.js
index 22582c1..e25d091 100644
--- a/modules/web-console/backend/services/domains.js
+++ b/modules/web-console/backend/services/domains.js
@@ -17,22 +17,23 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/domains',
- inject: ['require(lodash)', 'mongo', 'services/spaces', 'services/caches', 'errors']
+ inject: ['mongo', 'services/spaces', 'services/caches', 'errors']
};
/**
- * @param _
* @param mongo
* @param {SpacesService} spacesService
* @param {CachesService} cachesService
* @param errors
* @returns {DomainsService}
*/
-module.exports.factory = (_, mongo, spacesService, cachesService, errors) => {
+module.exports.factory = (mongo, spacesService, cachesService, errors) => {
/**
* Convert remove status operation to own presentation.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/downloads.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/downloads.js b/modules/web-console/backend/services/downloads.js
index 3dfc2be..6a03fe2 100644
--- a/modules/web-console/backend/services/downloads.js
+++ b/modules/web-console/backend/services/downloads.js
@@ -17,24 +17,25 @@
'use strict';
+const fs = require('fs');
+const path = require('path');
+const _ = require('lodash');
+const JSZip = require('jszip');
+
// Fire me up!
module.exports = {
implements: 'services/agents',
- inject: ['require(lodash)', 'require(fs)', 'require(path)', 'require(jszip)', 'settings', 'agents-handler', 'errors']
+ inject: ['settings', 'agents-handler', 'errors']
};
/**
- * @param _
- * @param fs
- * @param path
- * @param JSZip
* @param settings
* @param agentsHnd
* @param errors
* @returns {DownloadsService}
*/
-module.exports.factory = (_, fs, path, JSZip, settings, agentsHnd, errors) => {
+module.exports.factory = (settings, agentsHnd, errors) => {
class DownloadsService {
/**
* Get agent archive with user agent configuration.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/igfss.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/igfss.js b/modules/web-console/backend/services/igfss.js
index acb565b..e9ff38e 100644
--- a/modules/web-console/backend/services/igfss.js
+++ b/modules/web-console/backend/services/igfss.js
@@ -17,21 +17,22 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/igfss',
- inject: ['require(lodash)', 'mongo', 'services/spaces', 'errors']
+ inject: ['mongo', 'services/spaces', 'errors']
};
/**
- * @param _
* @param mongo
* @param {SpacesService} spacesService
* @param errors
* @returns {IgfssService}
*/
-module.exports.factory = (_, mongo, spacesService, errors) => {
+module.exports.factory = (mongo, spacesService, errors) => {
/**
* Convert remove status operation to own presentation.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/mails.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/mails.js b/modules/web-console/backend/services/mails.js
index 3c22a95..6a31e93 100644
--- a/modules/web-console/backend/services/mails.js
+++ b/modules/web-console/backend/services/mails.js
@@ -17,20 +17,21 @@
'use strict';
+const _ = require('lodash');
+const nodemailer = require('nodemailer');
+
// Fire me up!
module.exports = {
implements: 'services/mails',
- inject: ['require(lodash)', 'require(nodemailer)', 'settings']
+ inject: ['settings']
};
/**
- * @param _
- * @param nodemailer
* @param settings
* @returns {MailsService}
*/
-module.exports.factory = (_, nodemailer, settings) => {
+module.exports.factory = (settings) => {
/**
* Send mail to user.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/notebooks.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/notebooks.js b/modules/web-console/backend/services/notebooks.js
index 9aa2c38..5d8b57d 100644
--- a/modules/web-console/backend/services/notebooks.js
+++ b/modules/web-console/backend/services/notebooks.js
@@ -17,21 +17,22 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/notebooks',
- inject: ['require(lodash)', 'mongo', 'services/spaces', 'errors']
+ inject: ['mongo', 'services/spaces', 'errors']
};
/**
- * @param _
* @param mongo
* @param {SpacesService} spacesService
* @param errors
* @returns {NotebooksService}
*/
-module.exports.factory = (_, mongo, spacesService, errors) => {
+module.exports.factory = (mongo, spacesService, errors) => {
/**
* Convert remove status operation to own presentation.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/notifications.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/notifications.js b/modules/web-console/backend/services/notifications.js
index f1860e7..c7d8033 100644
--- a/modules/web-console/backend/services/notifications.js
+++ b/modules/web-console/backend/services/notifications.js
@@ -21,16 +21,15 @@
module.exports = {
implements: 'services/notifications',
- inject: ['require(lodash)', 'mongo', 'browsers-handler']
+ inject: ['mongo', 'browsers-handler']
};
/**
- * @param _
* @param mongo
* @param browsersHnd
* @returns {NotificationsService}
*/
-module.exports.factory = (_, mongo, browsersHnd) => {
+module.exports.factory = (mongo, browsersHnd) => {
class NotificationsService {
/**
* Update notifications.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/sessions.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/sessions.js b/modules/web-console/backend/services/sessions.js
index 7f3fc73..0518ce2 100644
--- a/modules/web-console/backend/services/sessions.js
+++ b/modules/web-console/backend/services/sessions.js
@@ -21,16 +21,15 @@
module.exports = {
implements: 'services/sessions',
- inject: ['require(lodash)', 'mongo', 'errors']
+ inject: ['mongo', 'errors']
};
/**
- * @param _
* @param mongo
* @param errors
* @returns {SessionsService}
*/
-module.exports.factory = (_, mongo, errors) => {
+module.exports.factory = (mongo, errors) => {
class SessionsService {
/**
* Become user.
http://git-wip-us.apache.org/repos/asf/ignite/blob/c10aa0c4/modules/web-console/backend/services/users.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 991928a..620f7e9 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -17,15 +17,16 @@
'use strict';
+const _ = require('lodash');
+
// Fire me up!
module.exports = {
implements: 'services/users',
- inject: ['require(lodash)', 'errors', 'settings', 'mongo', 'services/spaces', 'services/mails', 'services/activities', 'agents-handler']
+ inject: ['errors', 'settings', 'mongo', 'services/spaces', 'services/mails', 'services/activities', 'agents-handler']
};
/**
- * @param _
* @param mongo
* @param errors
* @param settings
@@ -35,7 +36,7 @@ module.exports = {
* @param {AgentsHandler} agentHnd
* @returns {UsersService}
*/
-module.exports.factory = (_, errors, settings, mongo, spacesService, mailsService, activitiesService, agentHnd) => {
+module.exports.factory = (errors, settings, mongo, spacesService, mailsService, activitiesService, agentHnd) => {
const _randomString = () => {
const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
const possibleLen = possible.length;
[18/50] [abbrv] ignite git commit: IGNITE-7172 Minor fix.
Posted by sb...@apache.org.
IGNITE-7172 Minor fix.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/802a1668
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/802a1668
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/802a1668
Branch: refs/heads/ignite-zk
Commit: 802a1668b7586048f42a726ba7e69854fb607b2e
Parents: 9ca3c09
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Dec 14 16:12:03 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Dec 14 16:12:03 2017 +0700
----------------------------------------------------------------------
.../frontend/app/components/page-queries/template.tpl.pug | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/802a1668/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
index ce5ab73..1574874 100644
--- a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
@@ -364,7 +364,7 @@ mixin paragraph-query
.row(ng-if='notebook' bs-affix style='margin-bottom: 20px;')
+notebook-rename
- -var example = `CREATE TABLE Person(ID INTEGER PRIMARY KEY, NAME VARCHAR(100));\nINSERT INTO Person(ID, NAME) VALUES (1, "Ed"), (2, "Ann"), (3, "Emma");\nSELECT * FROM Person;`;
+ -var example = `CREATE TABLE Person(ID INTEGER PRIMARY KEY, NAME VARCHAR(100));\nINSERT INTO Person(ID, NAME) VALUES (1, 'Ed'), (2, 'Ann'), (3, 'Emma');\nSELECT * FROM Person;`;
ignite-information(
data-title='With query notebook you can'
[05/50] [abbrv] ignite git commit: IGNITE-6390 Web Console: Fixed
cluster selector on topology changed.
Posted by sb...@apache.org.
IGNITE-6390 Web Console: Fixed cluster selector on topology changed.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e6489ea1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e6489ea1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e6489ea1
Branch: refs/heads/ignite-zk
Commit: e6489ea18fe4aee37ee8c4ade4c24167038ddd73
Parents: e3bbc98
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Dec 13 10:58:22 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Dec 13 10:58:22 2017 +0700
----------------------------------------------------------------------
.../frontend/app/components/page-queries/controller.js | 5 +----
.../frontend/app/modules/agent/AgentManager.service.js | 6 ++++++
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e6489ea1/modules/web-console/frontend/app/components/page-queries/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/controller.js b/modules/web-console/frontend/app/components/page-queries/controller.js
index dba0269..bd03b22 100644
--- a/modules/web-console/frontend/app/components/page-queries/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/controller.js
@@ -929,9 +929,6 @@ export default class {
const awaitClusters$ = fromPromise(
agentMgr.startClusterWatch('Back to Configuration', 'base.configuration.tabs.advanced.clusters'));
- const currentCluster$ = agentMgr.connectionSbj
- .distinctUntilChanged((n, o) => n.cluster === o.cluster);
-
const finishLoading$ = defer(() => {
if (!$root.IgniteDemoMode)
Loading.finish('sqlLoading');
@@ -942,7 +939,7 @@ export default class {
};
this.refresh$ = awaitClusters$
- .mergeMap(() => currentCluster$)
+ .mergeMap(() => agentMgr.currentCluster$)
.do(() => Loading.start('sqlLoading'))
.do(() => {
_.forEach($scope.notebook.paragraphs, (paragraph) => {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e6489ea1/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 7668132..7bc23a9 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -112,6 +112,12 @@ export default class IgniteAgentManager {
this.connectionSbj = new BehaviorSubject(new ConnectionState(cluster));
+ let prevCluster;
+
+ this.currentCluster$ = this.connectionSbj
+ .distinctUntilChanged(({ cluster }) => prevCluster === cluster)
+ .do(({ cluster }) => prevCluster = cluster);
+
this.clusterVersion = '2.1.0';
if (!this.isDemoMode()) {
[12/50] [abbrv] ignite git commit: IGNITE-4943 Web Console: Improved
tooltips on "Admin panel" screen.
Posted by sb...@apache.org.
IGNITE-4943 Web Console: Improved tooltips on "Admin panel" screen.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c9d6dd5f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c9d6dd5f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c9d6dd5f
Branch: refs/heads/ignite-zk
Commit: c9d6dd5f4dc516e327eac963d458adc5bf4c26ec
Parents: a0516f9
Author: vsisko <vs...@gridgain.com>
Authored: Thu Dec 14 10:37:39 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Dec 14 10:37:39 2017 +0700
----------------------------------------------------------------------
.../list-of-registered-users.column-defs.js | 49 +++++++++++---------
1 file changed, 26 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c9d6dd5f/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
index 493c239..83382e0 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.column-defs.js
@@ -17,7 +17,8 @@
const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
-const USER_TEMPLATE = '<div class="ui-grid-cell-contents"><i class="pull-left" ng-class="row.entity.admin ? \'icon-admin\' : \'icon-user\'"></i> {{ COL_FIELD }}</div>';
+const USER_TEMPLATE = '<div class="ui-grid-cell-contents">' +
+ '<i class="pull-left" ng-class="row.entity.admin ? \'icon-admin\' : \'icon-user\'"></i> <label bs-tooltip data-title="{{ COL_FIELD }}">{{ COL_FIELD }}</label></div>';
const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='icon-cluster'></i>${ICON_SORT}</div>`;
const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-object-group'></i>${ICON_SORT}</div>`;
@@ -46,35 +47,37 @@ const ACTIONS_TEMPLATE = `
</li>
</div>`;
-const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
+const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a bs-tooltip data-title="{{ COL_FIELD }}" ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
+const DATE_WITH_TITLE = '<div class="ui-grid-cell-contents"><label bs-tooltip data-title="{{ COL_FIELD | date:\'M/d/yy HH:mm\' }}">{{ COL_FIELD | date:"M/d/yy HH:mm" }}</label></div>';
+const VALUE_WITH_TITLE = '<div class="ui-grid-cell-contents"><label bs-tooltip data-title="{{ COL_FIELD }}">{{ COL_FIELD }}</label></div>';
export default [
{name: 'actions', enableHiding: false, displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, visible: false},
{name: 'user', enableHiding: false, displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, pinnedLeft: true, filter: { placeholder: 'Filter by name...' }},
{name: 'email', displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
- {name: 'company', displayName: 'Company', categoryDisplayName: 'Company', field: 'company', minWidth: 180, enableFiltering: true, filter: { placeholder: 'Filter by company...' }},
- {name: 'country', displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by country...' }},
- {name: 'lastlogin', displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 135, width: 135, enableFiltering: false, visible: false},
- {name: 'lastactivity', displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellFilter: 'date:"M/d/yy HH:mm"', minWidth: 135, width: 135, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
+ {name: 'company', displayName: 'Company', categoryDisplayName: 'Company', field: 'company', cellTemplate: VALUE_WITH_TITLE, minWidth: 180, enableFiltering: true, filter: { placeholder: 'Filter by company...' }},
+ {name: 'country', displayName: 'Country', categoryDisplayName: 'Country', field: 'countryCode', cellTemplate: VALUE_WITH_TITLE, minWidth: 160, enableFiltering: true, filter: { placeholder: 'Filter by country...' }},
+ {name: 'lastlogin', displayName: 'Last login', categoryDisplayName: 'Last login', field: 'lastLogin', cellTemplate: DATE_WITH_TITLE, minWidth: 135, width: 135, enableFiltering: false, visible: false},
+ {name: 'lastactivity', displayName: 'Last activity', categoryDisplayName: 'Last activity', field: 'lastActivity', cellTemplate: DATE_WITH_TITLE, minWidth: 135, width: 135, enableFiltering: false, visible: true, sort: { direction: 'desc', priority: 0 }},
// Configurations
- {name: 'cfg_clusters', displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Clusters count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
- {name: 'cfg_models', displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Models count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
- {name: 'cfg_caches', displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Caches count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
- {name: 'cfg_igfs', displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'IGFS count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
+ {name: 'cfg_clusters', displayName: 'Clusters count', categoryDisplayName: 'Configurations', headerCellTemplate: CLUSTER_HEADER_TEMPLATE, field: 'counters.clusters', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Clusters count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
+ {name: 'cfg_models', displayName: 'Models count', categoryDisplayName: 'Configurations', headerCellTemplate: MODEL_HEADER_TEMPLATE, field: 'counters.models', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Models count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
+ {name: 'cfg_caches', displayName: 'Caches count', categoryDisplayName: 'Configurations', headerCellTemplate: CACHE_HEADER_TEMPLATE, field: 'counters.caches', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Caches count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
+ {name: 'cfg_igfs', displayName: 'IGFS count', categoryDisplayName: 'Configurations', headerCellTemplate: IGFS_HEADER_TEMPLATE, field: 'counters.igfs', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'IGFS count', minWidth: 65, width: 65, enableFiltering: false, visible: false},
// Activities Total
- {name: 'cfg', displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of configuration usages', minWidth: 70, width: 70, enableFiltering: false},
- {name: 'qry', displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of queries usages', minWidth: 70, width: 70, enableFiltering: false},
- {name: 'demo', displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of demo startup', minWidth: 85, width: 85, enableFiltering: false},
- {name: 'dnld', displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of agent downloads', minWidth: 80, width: 80, enableFiltering: false},
- {name: 'starts', displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of agent startup', minWidth: 87, width: 87, enableFiltering: false},
+ {name: 'cfg', displayName: 'Cfg', categoryDisplayName: 'Total activities', field: 'activitiesTotal["configuration"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of configuration usages', minWidth: 70, width: 70, enableFiltering: false},
+ {name: 'qry', displayName: 'Qry', categoryDisplayName: 'Total activities', field: 'activitiesTotal["queries"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of queries usages', minWidth: 70, width: 70, enableFiltering: false},
+ {name: 'demo', displayName: 'Demo', categoryDisplayName: 'Total activities', field: 'activitiesTotal["demo"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of demo startup', minWidth: 85, width: 85, enableFiltering: false},
+ {name: 'dnld', displayName: 'Dnld', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/download"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of agent downloads', minWidth: 80, width: 80, enableFiltering: false},
+ {name: 'starts', displayName: 'Starts', categoryDisplayName: 'Total activities', field: 'activitiesDetail["/agent/start"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Total count of agent startup', minWidth: 87, width: 87, enableFiltering: false},
// Activities Configuration
- {name: 'clusters', displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration clusters', minWidth: 100, width: 100, enableFiltering: false, visible: false},
- {name: 'model', displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration model', minWidth: 87, width: 87, enableFiltering: false, visible: false},
- {name: 'caches', displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration caches', minWidth: 96, width: 96, enableFiltering: false, visible: false},
- {name: 'igfs', displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration IGFS', minWidth: 85, width: 85, enableFiltering: false, visible: false},
- {name: 'summary', displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration summary', minWidth: 111, width: 111, enableFiltering: false, visible: false},
+ {name: 'clusters', displayName: 'Clusters', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/clusters"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration clusters', minWidth: 100, width: 100, enableFiltering: false, visible: false},
+ {name: 'model', displayName: 'Model', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/domains"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration model', minWidth: 87, width: 87, enableFiltering: false, visible: false},
+ {name: 'caches', displayName: 'Caches', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/caches"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration caches', minWidth: 96, width: 96, enableFiltering: false, visible: false},
+ {name: 'igfs', displayName: 'IGFS', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/igfs"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration IGFS', minWidth: 85, width: 85, enableFiltering: false, visible: false},
+ {name: 'summary', displayName: 'Summary', categoryDisplayName: 'Configuration\'s activities', field: 'activitiesDetail["/configuration/summary"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Configuration summary', minWidth: 111, width: 111, enableFiltering: false, visible: false},
// Activities Queries
- {name: 'execute', displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Query executions', minWidth: 98, width: 98, enableFiltering: false, visible: false},
- {name: 'explain', displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Query explain executions', minWidth: 95, width: 95, enableFiltering: false, visible: false},
- {name: 'scan', displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Scan query executions', minWidth: 80, width: 80, enableFiltering: false, visible: false}
+ {name: 'execute', displayName: 'Execute', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/execute"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Query executions', minWidth: 98, width: 98, enableFiltering: false, visible: false},
+ {name: 'explain', displayName: 'Explain', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/explain"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Query explain executions', minWidth: 95, width: 95, enableFiltering: false, visible: false},
+ {name: 'scan', displayName: 'Scan', categoryDisplayName: 'Queries\' activities', field: 'activitiesDetail["/queries/scan"] || 0', cellTemplate: VALUE_WITH_TITLE, type: 'number', cellClass: 'ui-grid-number-cell', headerTooltip: 'Scan query executions', minWidth: 80, width: 80, enableFiltering: false, visible: false}
];
[33/50] [abbrv] ignite git commit: IGNITE-7213: Empty class
descriptions for KNNModelFormat
Posted by sb...@apache.org.
IGNITE-7213: Empty class descriptions for KNNModelFormat
this closes #3242
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/91be7aff
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/91be7aff
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/91be7aff
Branch: refs/heads/ignite-zk
Commit: 91be7aff1050b2a62e42ae8ec16cdaf0f2de9629
Parents: 86dc36d
Author: YuriBabak <y....@gmail.com>
Authored: Fri Dec 15 20:22:32 2017 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Dec 15 20:22:32 2017 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/ml/knn/models/KNNModelFormat.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/91be7aff/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModelFormat.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModelFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModelFormat.java
index 17d9842..11a23f5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModelFormat.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/models/KNNModelFormat.java
@@ -22,7 +22,11 @@ import java.util.Arrays;
import org.apache.ignite.ml.math.distances.DistanceMeasure;
import org.apache.ignite.ml.structures.LabeledDataset;
-/** */
+/**
+ * kNN model representation.
+ *
+ * @see KNNModel
+ */
public class KNNModelFormat implements Serializable {
/** Amount of nearest neighbors. */
private int k;
[46/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e419ec83
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e419ec83
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e419ec83
Branch: refs/heads/ignite-zk
Commit: e419ec83df5a986a968cd5a63b5b1bb8cf76673f
Parents: 4525b92
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 19 14:51:43 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 19 15:00:39 2017 +0300
----------------------------------------------------------------------
.../DefaultCommunicationProblemResolver.java | 45 +++++++---
.../zk/internal/ZookeeperDiscoveryImpl.java | 9 +-
.../ZookeeperDiscoverySpiBasicTest.java | 89 ++++++++++++--------
.../testframework/junits/GridAbstractTest.java | 2 +-
4 files changed, 95 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e419ec83/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
index 1e973d3..ed2ddb8 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
@@ -20,6 +20,7 @@ package org.apache.ignite.configuration;
import java.util.BitSet;
import java.util.List;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.CU;
/**
*
@@ -44,6 +45,27 @@ public class DefaultCommunicationProblemResolver implements CommunicationProblem
/**
*
*/
+ private static class ClusterSearch {
+ /** */
+ int srvCnt;
+
+ /** */
+ int nodeCnt;
+
+ /** */
+ final BitSet nodesBitSet;
+
+ /**
+ * @param nodes Total nodes.
+ */
+ ClusterSearch(int nodes) {
+ nodesBitSet = new BitSet(nodes);
+ }
+ }
+
+ /**
+ *
+ */
private static class ClusterGraph {
/** */
private final static int WORD_IDX_SHIFT = 6;
@@ -95,26 +117,21 @@ public class DefaultCommunicationProblemResolver implements CommunicationProblem
* @return Cluster nodes bit set.
*/
BitSet findLargestIndependentCluster() {
- BitSet maxCluster = null;
- int maxClusterSize = 0;
+ ClusterSearch maxCluster = null;
for (int i = 0; i < nodeCnt; i++) {
if (getBit(visitBitSet, i))
continue;
- BitSet cluster = new BitSet(nodeCnt);
+ ClusterSearch cluster = new ClusterSearch(nodeCnt);
search(cluster, i);
- int size = cluster.cardinality();
-
- if (maxCluster == null || size > maxClusterSize) {
+ if (maxCluster == null || cluster.srvCnt > maxCluster.srvCnt)
maxCluster = cluster;
- maxClusterSize = size;
- }
}
- return maxCluster;
+ return maxCluster != null ? maxCluster.nodesBitSet : null;
}
/**
@@ -154,13 +171,19 @@ public class DefaultCommunicationProblemResolver implements CommunicationProblem
* @param cluster Current cluster bit set.
* @param idx Node index.
*/
- void search(BitSet cluster, int idx) {
+ void search(ClusterSearch cluster, int idx) {
+ assert !getBit(visitBitSet, idx);
+
setBit(visitBitSet, idx);
- cluster.set(idx);
+ cluster.nodesBitSet.set(idx);
+ cluster.nodeCnt++;
ClusterNode node1 = nodes.get(idx);
+ if (!CU.clientNode(node1))
+ cluster.srvCnt++;
+
for (int i = 0; i < nodeCnt; i++) {
if (i == idx || getBit(visitBitSet, i))
continue;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e419ec83/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index f1ad869..7032fd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -2088,7 +2088,7 @@ public class ZookeeperDiscoveryImpl {
assert node != null : msg.nodeId;
if (node.isLocal())
- throw localNodeFail("Received force EVT_NODE_FAILED event for local node.");
+ throw localNodeFail("Received force EVT_NODE_FAILED event for local node.", true);
else
notifyNodeFail(node.internalId(), evtData.topologyVersion());
}
@@ -2137,7 +2137,7 @@ public class ZookeeperDiscoveryImpl {
deleteAliveNodes(res.killedNodes);
throw localNodeFail("Local node is forced to stop by communication error resolver " +
- "[nodeId=" + locNode.id() + ']');
+ "[nodeId=" + locNode.id() + ']', false);
}
ZookeeperClusterNode node = rtState.top.nodesByInternalId.get(internalId);
@@ -2527,14 +2527,15 @@ public class ZookeeperDiscoveryImpl {
/**
* @param msg Message to log.
+ * @param clientReconnect {@code True} if allow client reconnect.
* @return Exception to be thrown.
*/
- private Exception localNodeFail(String msg) {
+ private Exception localNodeFail(String msg, boolean clientReconnect) {
U.warn(log, msg);
rtState.onCloseStart();
- if (clientReconnectEnabled) {
+ if (clientReconnect && clientReconnectEnabled) {
assert locNode.isClient() : locNode;
boolean reconnect = false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e419ec83/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index 44e48f9..0630af4 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -1718,23 +1718,6 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
- * TODO ZK
- *
- * @throws Exception If failed.
- */
- public void _testCommunicationFailure() throws Exception {
- Ignite srv0 = startGrid(0);
-
- Ignite srv1 = startGrid(1);
-
- info("Close communication");
-
- ((TcpCommunicationSpi)srv1.configuration().getCommunicationSpi()).simulateNodeFailure();
-
- Thread.sleep(60_000);
- }
-
- /**
* @throws Exception If failed.
*/
public void testPing() throws Exception {
@@ -2035,23 +2018,9 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
startGrids(3);
- {
- ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(0));
- commSpi.checkRes = new BitSet(3);
- commSpi.checkRes.set(0);
- commSpi.checkRes.set(1);
- }
- {
- ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(1));
- commSpi.checkRes = new BitSet(3);
- commSpi.checkRes.set(0);
- commSpi.checkRes.set(1);
- }
- {
- ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(2));
- commSpi.checkRes = new BitSet(3);
- commSpi.checkRes.set(2);
- }
+ ZkTestCommunicationSpi.forNode(ignite(0)).initCheckResult(3, 0, 1);
+ ZkTestCommunicationSpi.forNode(ignite(1)).initCheckResult(3, 0, 1);
+ ZkTestCommunicationSpi.forNode(ignite(0)).initCheckResult(3, 2);
UUID killedId = nodeId(2);
@@ -2069,6 +2038,47 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testDefaultCommunicationErrorResolver2() throws Exception {
+ testCommSpi = true;
+ sesTimeout = 5000;
+
+ startGrids(3);
+
+ client = true;
+
+ startGridsMultiThreaded(3, 2);
+
+ ZkTestCommunicationSpi.forNode(ignite(0)).initCheckResult(5, 0, 1);
+ ZkTestCommunicationSpi.forNode(ignite(1)).initCheckResult(5, 0, 1);
+ ZkTestCommunicationSpi.forNode(ignite(2)).initCheckResult(5, 2, 3, 4);
+ ZkTestCommunicationSpi.forNode(ignite(3)).initCheckResult(5, 2, 3, 4);
+ ZkTestCommunicationSpi.forNode(ignite(4)).initCheckResult(5, 2, 3, 4);
+
+ ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+ spi.resolveCommunicationError(spi.getNode(ignite(1).cluster().localNode().id()), new Exception("test"));
+
+ waitForTopology(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDefaultCommunicationErrorResolver3() throws Exception {
+ sesTimeout = 5000;
+
+ startGridsMultiThreaded(3);
+
+ info("Close communication");
+
+ ((TcpCommunicationSpi)ignite(1).configuration().getCommunicationSpi()).simulateNodeFailure();
+
+ waitForTopology(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testConnectionCheck() throws Exception {
final int NODES = 5;
@@ -2832,6 +2842,17 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
return (ZkTestCommunicationSpi)ignite.configuration().getCommunicationSpi();
}
+ /**
+ * @param nodes Number of nodes.
+ * @param setBitIdxs Bits indexes to set in check result.
+ */
+ void initCheckResult(int nodes, Integer... setBitIdxs) {
+ checkRes = new BitSet(nodes);
+
+ for (Integer bitIdx : setBitIdxs)
+ checkRes.set(bitIdx);
+ }
+
/** {@inheritDoc} */
@Override public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
CountDownLatch pingLatch = this.pingLatch;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e419ec83/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 8afbdb7..4b62534 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -2188,7 +2188,7 @@ public abstract class GridAbstractTest extends TestCase {
return true;
}
- }, 15_000));
+ }, 30_000));
}
/**
[35/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/fc085a4a
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/fc085a4a
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/fc085a4a
Branch: refs/heads/ignite-zk
Commit: fc085a4a018c1f90861a76842f3eebdeee0ba567
Parents: 8101455
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 18 10:49:43 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 18 12:50:53 2017 +0300
----------------------------------------------------------------------
.../managers/communication/GridIoManager.java | 4 +-
.../communication/GridIoMessageFactory.java | 12 ++++
.../ignite/internal/util/nio/GridNioServer.java | 59 +++++++++++--------
.../TcpCommunicationConnectionCheckFuture.java | 61 ++++++++++++++++----
.../internal/ZkCommunicationErrorNodeState.java | 44 ++++++++++++++
.../ZkCommunicationErrorProcessFuture.java | 41 ++++++++++---
.../ZkCommunicationErrorResolveResult.java | 3 +
.../ZkDistributedCollectDataFuture.java | 11 ++++
.../discovery/zk/internal/ZkRuntimeState.java | 5 ++
.../zk/internal/ZookeeperDiscoveryImpl.java | 37 ++++++++++--
.../ZookeeperDiscoverySpiBasicTest.java | 23 +++++---
11 files changed, 241 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 9c6271a..81f00e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -300,9 +300,9 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
@Override public MessageReader reader(UUID rmtNodeId, MessageFactory msgFactory)
throws IgniteCheckedException {
- assert rmtNodeId != null;
- return new DirectMessageReader(msgFactory, U.directProtocolVersion(ctx, rmtNodeId));
+ return new DirectMessageReader(msgFactory,
+ rmtNodeId != null ? U.directProtocolVersion(ctx, rmtNodeId) : GridIoManager.DIRECT_PROTO_VER);
}
};
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 78cb7a8..51a6e25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -117,6 +117,8 @@ import org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse;
import org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx;
+import org.apache.ignite.internal.processors.cluster.ClusterMetricsUpdateMessage;
+import org.apache.ignite.internal.processors.continuous.ContinuousRoutineStartResultMessage;
import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
@@ -879,6 +881,16 @@ public class GridIoMessageFactory implements MessageFactory {
break;
+ case 129:
+ msg = new ClusterMetricsUpdateMessage();
+
+ break;
+
+
+ case 130:
+ msg = new ContinuousRoutineStartResultMessage();
+
+ break;
// [-3..119] [124..128] [-23..-27] [-36..-55]- this
// [120..123] - DR
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 9784549..e95f957 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -2301,7 +2301,11 @@ public class GridNioServer<T> {
else if (log.isDebugEnabled())
log.debug("Failed to process selector key [ses=" + ses + ", err=" + e + ']');
- close(ses, new GridNioException(e));
+ // Can be null if async connect failed.
+ if (ses != null)
+ close(ses, new GridNioException(e));
+ else
+ closeKey(key);
}
}
}
@@ -2525,6 +2529,34 @@ public class GridNioServer<T> {
}
/**
+ * @param key Key.
+ */
+ private void closeKey(SelectionKey key) {
+ // Shutdown input and output so that remote client will see correct socket close.
+ Socket sock = ((SocketChannel)key.channel()).socket();
+
+ try {
+ try {
+ sock.shutdownInput();
+ }
+ catch (IOException ignored) {
+ // No-op.
+ }
+
+ try {
+ sock.shutdownOutput();
+ }
+ catch (IOException ignored) {
+ // No-op.
+ }
+ }
+ finally {
+ U.close(key, log);
+ U.close(sock, log);
+ }
+ }
+
+ /**
* Closes the session and all associated resources, then notifies the listener.
*
* @param ses Session to be closed.
@@ -2544,8 +2576,6 @@ public class GridNioServer<T> {
sessions.remove(ses);
workerSessions.remove(ses);
- SelectionKey key = ses.key();
-
if (ses.setClosed()) {
ses.onClosed();
@@ -2557,28 +2587,7 @@ public class GridNioServer<T> {
((DirectBuffer)ses.readBuffer()).cleaner().clean();
}
- // Shutdown input and output so that remote client will see correct socket close.
- Socket sock = ((SocketChannel)key.channel()).socket();
-
- try {
- try {
- sock.shutdownInput();
- }
- catch (IOException ignored) {
- // No-op.
- }
-
- try {
- sock.shutdownOutput();
- }
- catch (IOException ignored) {
- // No-op.
- }
- }
- finally {
- U.close(key, log);
- U.close(sock, log);
- }
+ closeKey(ses.key());
if (e != null)
filterChain.onExceptionCaught(ses, e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
index 6cb5622..170ee44 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
@@ -111,13 +111,6 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
resBitSet = new BitSet(nodes.size());
}
- /** {@inheritDoc} */
- @Override public void onEvent(Event evt) {
- assert evt instanceof DiscoveryEvent : evt;
- assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ;
-
- }
-
/**
* @param timeout Connect timeout.
*/
@@ -160,6 +153,8 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
MultipleAddressesConnectFuture fut = new MultipleAddressesConnectFuture(i);
fut.init(addrs);
+
+ futs[i] = fut;
}
}
else
@@ -171,7 +166,7 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
spi.getSpiContext().addLocalEventListener(this, EVT_NODE_LEFT, EVT_NODE_FAILED);
if (!isDone()) {
- endTime = System.currentTimeMillis() - timeout;
+ endTime = System.currentTimeMillis() + timeout;
spi.getSpiContext().addTimeoutObject(this);
}
@@ -211,8 +206,30 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
}
/** {@inheritDoc} */
+ @Override public void onEvent(Event evt) {
+ if (isDone())
+ return;
+
+ assert evt instanceof DiscoveryEvent : evt;
+ assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ;
+
+ UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
+
+ for (int i = 0; i < nodes.size(); i++) {
+ if (nodes.get(i).id().equals(nodeId)) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onNodeFailed();
+
+ return;
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void onTimeout() {
- if (!isDone())
+ if (isDone())
return;
ConnectFuture[] futs = this.futs;
@@ -230,6 +247,8 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
if (super.onDone(res, err)) {
spi.getSpiContext().removeTimeoutObject(this);
+ spi.getSpiContext().removeLocalEventListener(this);
+
return true;
}
@@ -244,6 +263,11 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
*
*/
void onTimeout();
+
+ /**
+ *
+ */
+ void onNodeFailed();
}
/**
@@ -325,11 +349,16 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
finish(nodeId(nodeIdx).equals(rmtNodeId));
}
+ /** {@inheritDoc} */
+ @Override public void onNodeFailed() {
+ cancel();
+ }
+
/**
* @param res Result.
* @return {@code True} if result was set by this call.
*/
- boolean finish(boolean res) {
+ public boolean finish(boolean res) {
if (connFutDoneUpdater.compareAndSet(this, 0, 1)) {
onStatusReceived(res);
@@ -369,6 +398,18 @@ public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Bit
}
/** {@inheritDoc} */
+ @Override public void onNodeFailed() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onNodeFailed();
+ }
+ }
+
+ /** {@inheritDoc} */
@Override public void onTimeout() {
SingleAddressConnectFuture[] futs = this.futs;
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
new file mode 100644
index 0000000..ddc310d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.BitSet;
+
+/**
+ *
+ */
+public class ZkCommunicationErrorNodeState implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private final BitSet commState;
+
+ /** */
+ private final Exception err;
+
+ /**
+ * @param commState Communication state.
+ * @param err Error if failed get communication state..
+ */
+ ZkCommunicationErrorNodeState(BitSet commState, Exception err) {
+ this.commState = commState;
+ this.err = err;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index d7d4bd1..a6294bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -17,6 +17,7 @@
package org.apache.ignite.spi.discovery.zk.internal;
+import java.util.BitSet;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -29,6 +30,8 @@ import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
@@ -141,19 +144,41 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
}
/**
- * @param locNodeOrder Local node order.
* @param rtState Runtime state.
* @param futPath Future path.
* @param nodes Nodes to ping.
- * @throws Exception If failed.
*/
- void pingNodesAndNotifyFuture(long locNodeOrder, ZkRuntimeState rtState, String futPath, List<ClusterNode> nodes)
- throws Exception {
- TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
-
- spi.checkConnection(nodes);
+ void pingNodesAndNotifyFuture(final ZkRuntimeState rtState, final String futPath, List<ClusterNode> nodes) throws Exception {
+ final TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
+
+ IgniteFuture<BitSet> fut = spi.checkConnection(nodes);
+
+ fut.listen(new IgniteInClosure<IgniteFuture<BitSet>>() {
+ @Override public void apply(final IgniteFuture<BitSet> fut) {
+ // Future completed either from NIO thread or timeout worker, save result from another thread.
+ impl.runInWorkerThread(new ZkRunnable(rtState, impl) {
+ @Override public void run0() throws Exception {
+ BitSet commState = null;
+ Exception err = null;
+
+ try {
+ commState = fut.get();
+ }
+ catch (Exception e) {
+ err = e;
+ }
+
+ ZkCommunicationErrorNodeState state = new ZkCommunicationErrorNodeState(commState, err);
+
+ ZkDistributedCollectDataFuture.saveNodeResult(futPath,
+ rtState.zkClient,
+ impl.localNode().order(),
+ impl.marshalZip(state));
+ }
+ });
- ZkDistributedCollectDataFuture.saveNodeResult(futPath, rtState.zkClient, locNodeOrder, null);
+ }
+ });
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
index 745496b..607f93b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
@@ -31,6 +31,9 @@ class ZkCommunicationErrorResolveResult implements Serializable {
/** */
final GridLongList failedNodes;
+ /**
+ * @param failedNodes
+ */
ZkCommunicationErrorResolveResult(@Nullable GridLongList failedNodes) {
this.failedNodes = failedNodes;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
index e5d2356..19e2acc 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -107,6 +107,17 @@ class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
}
/**
+ * @param futPath
+ * @param client
+ * @param nodeOrder
+ * @return Node result data.
+ * @throws Exception If fai.ed
+ */
+ static byte[] readNodeResult(String futPath, ZookeeperClient client, long nodeOrder) throws Exception {
+ return client.getData(futPath + "/" + nodeOrder);
+ }
+
+ /**
* @param futResPath Result path.
* @param client Client.
* @param data Result data.
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
index fc03f8d..dc7b1bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
@@ -17,6 +17,8 @@
package org.apache.ignite.spi.discovery.zk.internal;
+import java.util.List;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.zookeeper.AsyncCallback;
import org.apache.zookeeper.Watcher;
@@ -73,6 +75,9 @@ class ZkRuntimeState {
/** */
final ZkClusterNodes top = new ZkClusterNodes();
+ /** */
+ List<ClusterNode> commErrProcNodes;
+
/**
* @param prevJoined {@code True} if joined topology before reconnect attempt.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 65bf6e7..62fc581 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -2056,6 +2056,8 @@ public class ZookeeperDiscoveryImpl {
rtState.evtsData.communicationErrorResolveFutureId(null);
+ rtState.commErrProcNodes = null;
+
ZkCommunicationErrorResolveResult res = msg.res;
if (res == null)
@@ -2120,7 +2122,8 @@ public class ZookeeperDiscoveryImpl {
final String futPath = zkPaths.distributedFutureBasePath(msg.id);
final ZkCommunicationErrorProcessFuture fut0 = fut;
- final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+ rtState.commErrProcNodes = rtState.top.topologySnapshot();
if (rtState.crd) {
ZkDistributedCollectDataFuture nodeResFut = collectCommunicationStatusFuture(msg.id);
@@ -2130,7 +2133,7 @@ public class ZookeeperDiscoveryImpl {
runInWorkerThread(new ZkRunnable(rtState, this) {
@Override protected void run0() throws Exception {
- fut0.pingNodesAndNotifyFuture(locNode.order(), rtState, futPath, topSnapshot);
+ fut0.pingNodesAndNotifyFuture(rtState, futPath, rtState.commErrProcNodes);
}
});
}
@@ -2145,7 +2148,7 @@ public class ZookeeperDiscoveryImpl {
new Callable<Void>() {
@Override public Void call() throws Exception {
// Future is completed from ZK event thread.
- onCommunicationResolveStatusReceived(rtState);
+ onCommunicationErrorResolveStatusReceived(rtState);
return null;
}
@@ -2157,16 +2160,38 @@ public class ZookeeperDiscoveryImpl {
* @param rtState Runtime state.
* @throws Exception If failed.
*/
- private void onCommunicationResolveStatusReceived(ZkRuntimeState rtState) throws Exception {
+ private void onCommunicationErrorResolveStatusReceived(ZkRuntimeState rtState) throws Exception {
ZkDiscoveryEventsData evtsData = rtState.evtsData;
UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
if (log.isInfoEnabled())
- log.info("Received communication status from all nodes, call resolver [reqId=" + futId + ']');
+ log.info("Received communication status from all nodes [reqId=" + futId + ']');
assert futId != null;
+ String futPath = zkPaths.distributedFutureBasePath(futId);
+
+ List<ClusterNode> initialNodes = rtState.commErrProcNodes;
+
+ assert initialNodes != null;
+
+ rtState.commErrProcNodes = null;
+
+ ZkClusterNodes top = rtState.top;
+
+ List<ZkCommunicationErrorNodeState> nodesRes = new ArrayList<>();
+
+ for (ZookeeperClusterNode node : top.nodesByOrder.values()) {
+ byte[] stateBytes = ZkDistributedCollectDataFuture.readNodeResult(futPath,
+ rtState.zkClient,
+ node.order());
+
+ ZkCommunicationErrorNodeState nodeState = unmarshalZip(stateBytes);
+
+ nodesRes.add(nodeState);
+ }
+
ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId);
ZkCommunicationErrorResolveResult res = new ZkCommunicationErrorResolveResult(null);
@@ -2663,7 +2688,7 @@ public class ZookeeperDiscoveryImpl {
* @return Bytes.
* @throws IgniteCheckedException If failed.
*/
- private byte[] marshalZip(Object obj) throws IgniteCheckedException {
+ byte[] marshalZip(Object obj) throws IgniteCheckedException {
assert obj != null;
return U.zip(marsh.marshal(obj));
http://git-wip-us.apache.org/repos/asf/ignite/blob/fc085a4a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index a2e8784..cee2e76 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -1916,20 +1916,27 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
- * TODO ZK: move to comm spi tests.
- *
* @throws Exception If failed.
*/
- public void testNodesPing() throws Exception {
- startGrids(3);
+ public void testConnectionCheck() throws Exception {
+ final int NODES = 5;
+
+ startGridsMultiThreaded(NODES);
+
+ for (int i = 0; i < NODES; i++) {
+ Ignite node = ignite(i);
+
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)node.configuration().getCommunicationSpi();
- TcpCommunicationSpi spi = (TcpCommunicationSpi)ignite(1).configuration().getCommunicationSpi();
+ List<ClusterNode> nodes = new ArrayList<>();
- List<ClusterNode> nodes = new ArrayList<>();
+ nodes.addAll(node.cluster().nodes());
- nodes.add(ignite(2).cluster().localNode());
+ BitSet res = spi.checkConnection(nodes).get();
- // spi.pingNodes(nodes);
+ for (int j = 0; j < NODES; j++)
+ assertTrue(res.get(j));
+ }
}
/**
[22/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cbe9980b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cbe9980b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cbe9980b
Branch: refs/heads/ignite-zk
Commit: cbe9980b45d6135ef6f67610f751c1599df9e54a
Parents: 7e8f85f
Author: sboikov <sb...@gridgain.com>
Authored: Thu Dec 14 14:48:26 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Dec 14 18:04:38 2017 +0300
----------------------------------------------------------------------
.../CommunicationProblemContext.java | 62 ++++++++++
.../CommunicationProblemResolver.java | 28 +++++
.../configuration/IgniteConfiguration.java | 13 ++
.../ignite/internal/util/nio/GridNioServer.java | 10 +-
.../communication/tcp/TcpCommunicationSpi.java | 49 ++++++++
.../discovery/CommunicationProblemContext.java | 62 ----------
.../discovery/CommunicationProblemResolver.java | 28 -----
.../DefaultCommunicationProblemResolver.java | 2 +
.../ZkCommunicationErrorProcessFuture.java | 11 +-
.../internal/ZkCommunicationProblemContext.java | 65 ++++++++++
.../ZkDistributedCollectDataFuture.java | 6 +-
.../zk/internal/ZookeeperDiscoveryImpl.java | 59 ++++++++--
.../ZookeeperDiscoverySpiBasicTest.java | 118 ++++++++++++++++++-
13 files changed, 401 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemContext.java b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemContext.java
new file mode 100644
index 0000000..9d53a97
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemContext.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+
+/**
+ *
+ */
+public interface CommunicationProblemContext {
+ /**
+ * @return Current topology snapshot.
+ */
+ public List<ClusterNode> topologySnapshot();
+
+ /**
+ * @param node1 First node.
+ * @param node2 Second node.
+ * @return {@code True} if {@link CommunicationSpi} is able to establish connection from first node to second node.
+ */
+ public boolean connectionAvailable(ClusterNode node1, ClusterNode node2);
+
+ /**
+ * @return List of currently started cache.
+ */
+ public List<String> startedCaches();
+
+ /**
+ * @param cacheName Cache name.
+ * @return Cache partitions affinity assignment.
+ */
+ public List<List<ClusterNode>> cacheAffinity(String cacheName);
+
+ /**
+ * @param cacheName Cache name.
+ * @return Cache partitions owners.
+ */
+ public List<List<ClusterNode>> cachePartitionOwners(String cacheName);
+
+ /**
+ * @param node Node to kill.
+ */
+ public void killNode(ClusterNode node);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemResolver.java
new file mode 100644
index 0000000..d1c6f27
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationProblemResolver.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+/**
+ *
+ */
+public interface CommunicationProblemResolver {
+ /**
+ * @param ctx Context.
+ */
+ public void resolve(CommunicationProblemContext ctx);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index fc1fb6b..8c3c818 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -480,6 +480,9 @@ public class IgniteConfiguration {
/** Client connector configuration. */
private ClientConnectorConfiguration cliConnCfg = ClientListenerProcessor.DFLT_CLI_CFG;
+ /** */
+ private CommunicationProblemResolver commProblemRslvr;
+
/**
* Creates valid grid configuration with all default values.
*/
@@ -507,6 +510,8 @@ public class IgniteConfiguration {
loadBalancingSpi = cfg.getLoadBalancingSpi();
indexingSpi = cfg.getIndexingSpi();
+ commProblemRslvr = cfg.getCommunicationProblemResolver();
+
/*
* Order alphabetically for maintenance purposes.
*/
@@ -590,6 +595,14 @@ public class IgniteConfiguration {
warmupClos = cfg.getWarmupClosure();
}
+ public CommunicationProblemResolver getCommunicationProblemResolver() {
+ return commProblemRslvr;
+ }
+
+ public void setCommunicationProblemResolver(CommunicationProblemResolver commProblemRslvr) {
+ this.commProblemRslvr = commProblemRslvr;
+ }
+
/**
* Gets optional grid name. Returns {@code null} if non-default grid name was not
* provided.
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 1d595d2..14d55d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -485,6 +485,14 @@ public class GridNioServer<T> {
return fut;
}
+ public void closeFromWorkerThread(GridNioSession ses) {
+ assert ses instanceof GridSelectorNioSessionImpl : ses;
+
+ GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
+
+ ((AbstractNioClientWorker)ses0.worker()).close((GridSelectorNioSessionImpl)ses, null);
+ }
+
/**
* @param ses Session.
* @param msg Message.
@@ -834,7 +842,7 @@ public class GridNioServer<T> {
NioOperationFuture<GridNioSession> req = new NioOperationFuture<>(ch, false, meta);
if (async) {
- assert meta != null;
+ // assert meta != null;
req.op = NioOperation.CONNECT;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 0b6daa3..e815312 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -32,6 +32,7 @@ import java.nio.channels.SocketChannel;
import java.nio.channels.spi.AbstractInterruptibleChannel;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -81,6 +82,7 @@ import org.apache.ignite.internal.util.nio.GridConnectionBytesVerifyFilter;
import org.apache.ignite.internal.util.nio.GridDirectParser;
import org.apache.ignite.internal.util.nio.GridNioCodecFilter;
import org.apache.ignite.internal.util.nio.GridNioFilter;
+import org.apache.ignite.internal.util.nio.GridNioFuture;
import org.apache.ignite.internal.util.nio.GridNioMessageReaderFactory;
import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
import org.apache.ignite.internal.util.nio.GridNioMessageWriterFactory;
@@ -2564,6 +2566,53 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
sendMessage0(node, msg, null);
}
+ public IgniteFuture<BitSet> pingNodes(List<ClusterNode> nodes) {
+ ClusterNode node = nodes.get(0);
+
+ try {
+ LinkedHashSet<InetSocketAddress> addrs = nodeAddresses(node);
+
+ // /172.25.4.90:45012
+
+ for (InetSocketAddress addr : addrs) {
+ SocketChannel ch = SocketChannel.open();
+
+ ch.configureBlocking(false);
+
+ ch.socket().setTcpNoDelay(tcpNoDelay);
+ ch.socket().setKeepAlive(true);
+
+ boolean connect = ch.connect(addr);
+
+ if (!connect) {
+ GridNioFuture<GridNioSession> fut = nioSrvr.createSession(ch, null, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
+ @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
+ try {
+ GridNioSession ses = fut.get();
+
+ log.info("Ping connected");
+
+ nioSrvr.closeFromWorkerThread(ses);
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ });
+
+ fut.get();
+ }
+ else
+ log.info("Connected");
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteSpiException(e);
+ }
+
+ return null;
+ }
+
/**
* Sends given message to destination node. Note that characteristics of the
* exchange such as durability, guaranteed delivery or error notification is
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemContext.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemContext.java
deleted file mode 100644
index 71673f1..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemContext.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery;
-
-import java.util.List;
-import java.util.UUID;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.spi.communication.CommunicationSpi;
-
-/**
- *
- */
-public interface CommunicationProblemContext {
- /**
- * @return Current topology snapshot.
- */
- public List<ClusterNode> topologySnapshot();
-
- /**
- * @param node1 First node.
- * @param node2 Second node.
- * @return {@code True} if {@link CommunicationSpi} is able to establish connection from first node to second node.
- */
- public boolean connectionAvailable(ClusterNode node1, ClusterNode node2);
-
- /**
- * @return List of currently started cache.
- */
- public List<String> startedCaches();
-
- /**
- * @param cacheName Cache name.
- * @return Cache partitions affinity assignment.
- */
- public List<List<ClusterNode>> cacheAffinity(String cacheName);
-
- /**
- * @param cacheName Cache name.
- * @return Cache partitions owners.
- */
- public List<List<ClusterNode>> cachePartitionOwners(String cacheName);
-
- /**
- * @param node Node to kill.
- */
- public void killNode(ClusterNode node);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemResolver.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemResolver.java
deleted file mode 100644
index a9b620b..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/CommunicationProblemResolver.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery;
-
-/**
- *
- */
-public interface CommunicationProblemResolver {
- /**
- * @param ctx Context.
- */
- public void resolve(CommunicationProblemContext ctx);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
index 4d0262d..b2d4bf0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
@@ -20,6 +20,8 @@ package org.apache.ignite.spi.discovery;
import java.util.BitSet;
import java.util.List;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CommunicationProblemContext;
+import org.apache.ignite.configuration.CommunicationProblemResolver;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index 15744a2..6812ab0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -18,6 +18,7 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
@@ -30,11 +31,13 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.jboss.netty.util.internal.ConcurrentHashMap;
import org.jetbrains.annotations.Nullable;
/**
- *
+ * Future is created on each node when either connection error occurs or resolve communication error request
+ * received.
*/
class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implements IgniteSpiTimeoutObject, Runnable {
/** */
@@ -144,8 +147,12 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
* @param nodes Nodes to ping.
* @throws Exception If failed.
*/
- void pingNodesAndNotifyFuture(long locNodeOrder, ZkRuntimeState rtState, String futPath, Collection<ClusterNode> nodes)
+ void pingNodesAndNotifyFuture(long locNodeOrder, ZkRuntimeState rtState, String futPath, List<ClusterNode> nodes)
throws Exception {
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
+
+ spi.pingNodes(nodes);
+
ZkDistributedCollectDataFuture.saveNodeResult(futPath, rtState.zkClient, locNodeOrder, null);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
new file mode 100644
index 0000000..fd11b55
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CommunicationProblemContext;
+
+/**
+ *
+ */
+class ZkCommunicationProblemContext implements CommunicationProblemContext {
+ /** */
+ private Set<ClusterNode> killedNodes = new HashSet<>();
+
+ /** {@inheritDoc} */
+ @Override public List<ClusterNode> topologySnapshot() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean connectionAvailable(ClusterNode node1, ClusterNode node2) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<String> startedCaches() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> cacheAffinity(String cacheName) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public List<List<ClusterNode>> cachePartitionOwners(String cacheName) {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void killNode(ClusterNode node) {
+ if (node == null)
+ throw new NullPointerException();
+
+ killedNodes.add(node);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
index d33001b..e5d2356 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -78,8 +78,10 @@ class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
if (remainingNodes.isEmpty())
completeAndNotifyListener();
else {
- if (log.isInfoEnabled())
- log.info("Initialize data collect future [futPath=" + futPath + ", nodes=" + remainingNodes.size() + ']');
+ if (log.isInfoEnabled()) {
+ log.info("Initialize data collect future [futPath=" + futPath + ", " +
+ "remainingNodes=" + remainingNodes.size() + ']');
+ }
rtState.zkClient.getChildrenAsync(futPath, watcher, watcher);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 82d9c4b..65bf6e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -41,6 +41,7 @@ import org.apache.ignite.IgniteInterruptedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CommunicationProblemResolver;
import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.ClusterMetricsSnapshot;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
@@ -1043,6 +1044,22 @@ public class ZookeeperDiscoveryImpl {
assert locNode.order() > 0 : locNode;
assert rtState.evtsData != null;
+ UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
+
+ if (futId != null) {
+ if (log.isInfoEnabled()) {
+ log.info("New discovery coordinator will handle already started cluster-wide communication " +
+ "error resolve [reqId=" + futId + ']');
+ }
+
+ ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+ ZkDistributedCollectDataFuture collectResFut = collectCommunicationStatusFuture(futId);
+
+ if (fut != null)
+ fut.nodeResultCollectFuture(collectResFut);
+ }
+
for (ZkDiscoveryEventData evtData : rtState.evtsData.evts.values())
evtData.initRemainingAcks(rtState.top.nodesByOrder.values());
@@ -2106,16 +2123,7 @@ public class ZookeeperDiscoveryImpl {
final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
if (rtState.crd) {
- ZkDistributedCollectDataFuture nodeResFut = new ZkDistributedCollectDataFuture(this, rtState, futPath,
- new Callable<Void>() {
- @Override public Void call() throws Exception {
- // Future is completed from ZK event thread.
- finishCommunicationResolveProcess(rtState);
-
- return null;
- }
- }
- );
+ ZkDistributedCollectDataFuture nodeResFut = collectCommunicationStatusFuture(msg.id);
fut.nodeResultCollectFuture(nodeResFut);
}
@@ -2128,14 +2136,35 @@ public class ZookeeperDiscoveryImpl {
}
/**
+ * @param futId Future ID.
+ * @return Future.
+ * @throws Exception If failed.
+ */
+ private ZkDistributedCollectDataFuture collectCommunicationStatusFuture(UUID futId) throws Exception {
+ return new ZkDistributedCollectDataFuture(this, rtState, zkPaths.distributedFutureBasePath(futId),
+ new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ // Future is completed from ZK event thread.
+ onCommunicationResolveStatusReceived(rtState);
+
+ return null;
+ }
+ }
+ );
+ }
+
+ /**
* @param rtState Runtime state.
* @throws Exception If failed.
*/
- private void finishCommunicationResolveProcess(ZkRuntimeState rtState) throws Exception {
+ private void onCommunicationResolveStatusReceived(ZkRuntimeState rtState) throws Exception {
ZkDiscoveryEventsData evtsData = rtState.evtsData;
UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
+ if (log.isInfoEnabled())
+ log.info("Received communication status from all nodes, call resolver [reqId=" + futId + ']');
+
assert futId != null;
ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId);
@@ -2148,6 +2177,14 @@ public class ZookeeperDiscoveryImpl {
rtState.zkClient,
marshalZip(res));
+ CommunicationProblemResolver rslvr = spi.ignite().configuration().getCommunicationProblemResolver();
+
+ if (rslvr != null) {
+ ZkCommunicationProblemContext ctx = new ZkCommunicationProblemContext();
+
+ rslvr.resolve(ctx);
+ }
+
evtsData.evtIdGen++;
ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
http://git-wip-us.apache.org/repos/asf/ignite/blob/cbe9980b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index ec70be6..64fcd34 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -20,6 +20,7 @@ package org.apache.ignite.spi.discovery.zk.internal;
import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -66,13 +67,14 @@ import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.logger.java.JavaLogger;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.apache.ignite.spi.discovery.CommunicationProblemContext;
-import org.apache.ignite.spi.discovery.CommunicationProblemResolver;
+import org.apache.ignite.configuration.CommunicationProblemContext;
+import org.apache.ignite.configuration.CommunicationProblemResolver;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
import org.apache.ignite.testframework.GridTestUtils;
@@ -121,6 +123,9 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
private boolean testSockNio;
/** */
+ private boolean testCommSpi;
+
+ /** */
private int sesTimeout;
/** */
@@ -139,7 +144,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
private boolean persistence;
/** */
- private CommunicationProblemResolver communicationProblemResolver;
+ private CommunicationProblemResolver commProblemRslvr;
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -236,6 +241,12 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
cfg.setDataStorageConfiguration(memCfg);
}
+ if (testCommSpi)
+ cfg.setCommunicationSpi(new ZkTestCommunicationSpi());
+
+ if (commProblemRslvr != null)
+ cfg.setCommunicationProblemResolver(commProblemRslvr);
+
return cfg;
}
@@ -1792,7 +1803,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
assert nodes > 1;
sesTimeout = 2000;
- communicationProblemResolver = new NoOpCommunicationProblemResolver();
+ commProblemRslvr = new NoOpCommunicationProblemResolver();
startGridsMultiThreaded(nodes);
@@ -1824,7 +1835,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
public void testNoOpCommunicationErrorResolve_3() throws Exception {
// One node fails before sending communication status.
sesTimeout = 2000;
- communicationProblemResolver = new NoOpCommunicationProblemResolver();
+ commProblemRslvr = new NoOpCommunicationProblemResolver();
startGridsMultiThreaded(3);
@@ -1864,6 +1875,64 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_4() throws Exception {
+ // Coordinator changes while resolve process is in progress.
+ testCommSpi = true;
+
+ sesTimeout = 2000;
+ commProblemRslvr = new NoOpCommunicationProblemResolver();
+
+ startGrid(0);
+
+ startGridsMultiThreaded(1, 3);
+
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(3));
+
+ commSpi.pingLatch = new CountDownLatch(1);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() {
+ ZookeeperDiscoverySpi spi = spi(ignite(1));
+
+ spi.onCommunicationConnectionError(ignite(2).cluster().localNode(), new Exception("test"));
+
+ return null;
+ }
+ });
+
+ U.sleep(1000);
+
+ assertFalse(fut.isDone());
+
+ stopGrid(0);
+
+ commSpi.pingLatch.countDown();
+
+ fut.get();
+
+ waitForTopology(3);
+ }
+
+ /**
+ * TODO ZK: move to comm spi tests.
+ *
+ * @throws Exception If failed.
+ */
+ public void testNodesPing() throws Exception {
+ startGrids(3);
+
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)ignite(1).configuration().getCommunicationSpi();
+
+ List<ClusterNode> nodes = new ArrayList<>();
+
+ nodes.add(ignite(2).cluster().localNode());
+
+ spi.pingNodes(nodes);
+ }
+
+ /**
* @param dfltConsistenId Default consistent ID flag.
* @throws Exception If failed.
*/
@@ -2298,6 +2367,9 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}, 10000));
}
+ /**
+ * @param node Node.
+ */
private static void closeZkClient(Ignite node) {
DiscoverySpi spi = node.configuration().getDiscoverySpi();
@@ -2322,6 +2394,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
* @param spi Spi instance.
+ * @return Zookeeper client.
*/
private static ZooKeeper zkClient(ZookeeperDiscoverySpi spi) {
return GridTestUtils.getFieldValue(spi, "impl", "rtState", "zkClient", "zk");
@@ -2333,6 +2406,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
* @param log Logger.
* @param clients Clients.
* @param disconnectedC Closure which will be run when client node disconnected.
+ * @param closeSock {@code True} to simulate reconnect by closing zk client's socket.
* @throws Exception If failed.
*/
public static void reconnectClientNodes(final IgniteLogger log,
@@ -2449,7 +2523,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
* @param latch Latch.
* @throws Exception If failed.
*/
- protected static void waitReconnectEvent(IgniteLogger log, CountDownLatch latch) throws Exception {
+ private static void waitReconnectEvent(IgniteLogger log, CountDownLatch latch) throws Exception {
if (!latch.await(30_000, MILLISECONDS)) {
log.error("Failed to wait for reconnect event, will dump threads, latch count: " + latch.getCount());
@@ -2475,6 +2549,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
*
*/
+ @SuppressWarnings("MismatchedReadAndWriteOfArray")
static class TestAffinityFunction extends RendezvousAffinityFunction {
/** */
private static final long serialVersionUID = 0L;
@@ -2536,4 +2611,35 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
// No-op.
}
}
+
+ /**
+ *
+ */
+ static class ZkTestCommunicationSpi extends TcpCommunicationSpi {
+ /** */
+ private volatile CountDownLatch pingLatch;
+
+ /**
+ * @param ignite Node.
+ * @return Node's communication SPI.
+ */
+ static ZkTestCommunicationSpi forNode(Ignite ignite) {
+ return (ZkTestCommunicationSpi)ignite.configuration().getCommunicationSpi();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteFuture<BitSet> pingNodes(List<ClusterNode> nodes) {
+ CountDownLatch pingLatch = this.pingLatch;
+
+ try {
+ if (pingLatch != null)
+ pingLatch.await();
+ }
+ catch (InterruptedException e) {
+ throw new IgniteException(e);
+ }
+
+ return super.pingNodes(nodes);
+ }
+ }
}
[08/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/376a4845
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/376a4845
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/376a4845
Branch: refs/heads/ignite-zk
Commit: 376a484569db8d1a0eb4b43128d7e327defb5e3d
Parents: b7cbd4c
Author: sboikov <sb...@gridgain.com>
Authored: Wed Dec 13 16:00:34 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Dec 13 17:52:03 2017 +0300
----------------------------------------------------------------------
.../zk/internal/ZkAbstractCallabck.java | 66 +++
.../zk/internal/ZkAbstractChildrenCallback.java | 53 +++
.../zk/internal/ZkAbstractWatcher.java | 55 +++
.../zk/internal/ZkCollectDistributedFuture.java | 171 ++++++++
.../ZkCommunicationErrorProcessFuture.java | 225 ++++++++---
...kCommunicationErrorResolveFinishMessage.java | 38 ++
...ZkCommunicationErrorResolveStartMessage.java | 50 +++
.../zk/internal/ZkDiscoveryEventsData.java | 18 +
.../discovery/zk/internal/ZkEventAckFuture.java | 142 -------
.../zk/internal/ZkForceNodeFailMessage.java | 55 +++
.../discovery/zk/internal/ZkIgnitePaths.java | 8 +
.../ZkInternalCommunicationErrorMessage.java | 39 --
.../ZkInternalForceNodeFailMessage.java | 55 ---
.../spi/discovery/zk/internal/ZkRunnable.java | 51 +++
.../zk/internal/ZookeeperDiscoveryImpl.java | 397 +++++++++++--------
15 files changed, 981 insertions(+), 442 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
new file mode 100644
index 0000000..d2efb9f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+
+/**
+ *
+ */
+abstract class ZkAbstractCallabck {
+ /** */
+ final ZkRuntimeState rtState;
+
+ /** */
+ private final ZookeeperDiscoveryImpl impl;
+
+ /** */
+ private final GridSpinBusyLock busyLock;
+
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkAbstractCallabck(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ this.rtState = rtState;
+ this.impl = impl;
+
+ busyLock = impl.busyLock;
+ }
+
+ /**
+ * @return {@code True} if is able to start processing.
+ */
+ final boolean onProcessStart() {
+ return !rtState.closing && busyLock.enterBusy();
+ }
+
+ /**
+ *
+ */
+ final void onProcessEnd() {
+ busyLock.leaveBusy();
+ }
+
+ /**
+ * @param e Error.
+ */
+ final void onProcessError(Throwable e) {
+ impl.onFatalError(busyLock, e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
new file mode 100644
index 0000000..5679993
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+abstract class ZkAbstractChildrenCallback extends ZkAbstractCallabck implements AsyncCallback.Children2Callback {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkAbstractChildrenCallback(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ processResult0(rc, path, ctx, children, stat);
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ abstract void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
+ throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
new file mode 100644
index 0000000..9098d05
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+/**
+ *
+ */
+abstract class ZkAbstractWatcher extends ZkAbstractCallabck implements Watcher {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkAbstractWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public final void process(WatchedEvent evt) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ process0(evt);
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ /**
+ * @param evt Event.
+ * @throws Exception If failed.
+ */
+ protected abstract void process0(WatchedEvent evt) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java
new file mode 100644
index 0000000..fa529cf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkCollectDistributedFuture extends GridFutureAdapter<Void> {
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private final String futPath;
+
+ /** */
+ private final ZookeeperDiscoveryImpl impl;
+
+ /** */
+ private final Set<Long> remainingNodes;
+
+ /** */
+ private final Callable<Void> lsnr;
+
+ /**
+ * @param impl
+ * @param rtState
+ * @param futPath
+ */
+ ZkCollectDistributedFuture(ZookeeperDiscoveryImpl impl, ZkRuntimeState rtState, String futPath, Callable<Void> lsnr) throws Exception {
+ this.impl = impl;
+ this.log = impl.log();
+ this.futPath = futPath;
+ this.lsnr = lsnr;
+
+ ZkClusterNodes top = impl.nodes();
+
+ remainingNodes = U.newHashSet(top.nodesByOrder.size());
+
+ for (ZookeeperClusterNode node : top.nodesByInternalId.values())
+ remainingNodes.add(node.order());
+
+ NodeResultsWatcher watcher = new NodeResultsWatcher(rtState, impl);
+
+ if (remainingNodes.isEmpty())
+ completeAndNotifyListener();
+ else
+ rtState.zkClient.getChildrenAsync(futPath, watcher, watcher);
+ }
+
+ /**
+ * @throws Exception If listener call failed.
+ */
+ private void completeAndNotifyListener() throws Exception {
+ if (super.onDone())
+ lsnr.call();
+ }
+
+ /**
+ * @param futPath
+ * @param client
+ * @param nodeOrder
+ * @param data
+ * @throws Exception If failed.
+ */
+ static void saveNodeResult(String futPath, ZookeeperClient client, long nodeOrder, byte[] data) throws Exception {
+ client.createIfNeeded(futPath + "/" + nodeOrder, data, CreateMode.PERSISTENT);
+ }
+
+ /**
+ * @param node Failed node.
+ */
+ void onNodeFail(ZookeeperClusterNode node) throws Exception {
+ long nodeOrder = node.order();
+
+ if (remainingNodes.remove(nodeOrder)) {
+ int remaining = remainingNodes.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("ZkCollectDistributedFuture removed remaining failed node [node=" + nodeOrder +
+ ", remaining=" + remaining +
+ ", futPath=" + futPath + ']');
+ }
+
+ if (remaining == 0)
+ completeAndNotifyListener();
+ }
+ }
+
+ /**
+ *
+ */
+ class NodeResultsWatcher extends ZkAbstractWatcher implements AsyncCallback.Children2Callback {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ NodeResultsWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void process0(WatchedEvent evt) {
+ if (evt.getType() == Watcher.Event.EventType.NodeChildrenChanged)
+ impl.zkClient().getChildrenAsync(evt.getPath(), this, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ assert rc == 0 : KeeperException.Code.get(rc);
+
+ if (isDone())
+ return;
+
+ for (int i = 0; i < children.size(); i++) {
+ Long nodeOrder = Long.parseLong(children.get(i));
+
+ if (remainingNodes.remove(nodeOrder)) {
+ int remaining = remainingNodes.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("ZkCollectDistributedFuture added new result [node=" + nodeOrder +
+ ", remaining=" + remaining +
+ ", futPath=" + path + ']');
+ }
+
+ if (remaining == 0)
+ completeAndNotifyListener();
+ }
+ }
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index 2ea65e8..d87f500 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -17,24 +17,31 @@
package org.apache.ignite.spi.discovery.zk.internal;
-import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Callable;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.jetbrains.annotations.Nullable;
/**
*
*/
-class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements IgniteSpiTimeoutObject, Runnable {
+class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implements IgniteSpiTimeoutObject, Runnable {
/** */
private final ZookeeperDiscoveryImpl impl;
/** */
- private final Map<UUID, GridFutureAdapter<Boolean>> errNodes = new HashMap<>();
+ private final Map<Long, GridFutureAdapter<Boolean>> nodeFuts = new HashMap<>();
/** */
private final long endTime;
@@ -42,48 +49,142 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
/** */
private final IgniteUuid id;
+ /** */
+ private State state;
+
+ /** */
+ private long resolveTopVer;
+
+ /** */
+ private Set<Long> resFailedNodes;
+
+ /** */
+ private ZkCollectDistributedFuture nodeResFut;
+
+ /**
+ * @param impl Discovery impl.
+ * @param timeout Timeout to wait before initiating resolve process.
+ * @return Future.
+ */
+ static ZkCommunicationErrorProcessFuture createOnCommunicationError(ZookeeperDiscoveryImpl impl, long timeout) {
+ return new ZkCommunicationErrorProcessFuture(impl, State.WAIT_TIMEOUT, timeout);
+ }
+
+ /**
+ * @param impl Discovery impl.
+ * @return Future.
+ */
+ static ZkCommunicationErrorProcessFuture createOnStartResolveRequest(ZookeeperDiscoveryImpl impl) {
+ return new ZkCommunicationErrorProcessFuture(impl, State.RESOLVE_STARTED, 0);
+ }
+
/**
* @param impl Discovery implementation.
+ * @param state Initial state.
* @param timeout Wait timeout before initiating communication errors resolve.
*/
- ZkCommunicationErrorProcessFuture(ZookeeperDiscoveryImpl impl, long timeout) {
+ private ZkCommunicationErrorProcessFuture(ZookeeperDiscoveryImpl impl, State state, long timeout) {
+ assert state != State.DONE;
+
this.impl = impl;
- id = IgniteUuid.fromUuid(impl.localNode().id());
+ if (state == State.WAIT_TIMEOUT) {
+ assert timeout > 0 : timeout;
+
+ id = IgniteUuid.fromUuid(impl.localNode().id());
+ endTime = System.currentTimeMillis() + timeout;
+ }
+ else {
+ id = null;
+ endTime = 0;
+ }
- endTime = System.currentTimeMillis() + timeout;
+ this.state = state;
+ }
+
+ void nodeResultCollectFuture(ZkCollectDistributedFuture nodeResFut) {
+ assert nodeResFut == null : nodeResFut;
+
+ this.nodeResFut = nodeResFut;
+ }
+
+ void pingNodesAndNotifyFuture(long locNodeOrder, ZkRuntimeState rtState, String futPath, Collection<ClusterNode> nodes)
+ throws Exception {
+ ZkCollectDistributedFuture.saveNodeResult(futPath, rtState.zkClient, locNodeOrder, null);
}
/**
- * @param nodeId Node ID.
- * @return Future finished when communication error resolve is done.
+ *
*/
- GridFutureAdapter<Boolean> nodeStatusFuture(UUID nodeId) {
- GridFutureAdapter<Boolean> fut;
+ void scheduleCheckOnTimeout() {
+ synchronized (this) {
+ if (state == State.WAIT_TIMEOUT)
+ impl.spi.getSpiContext().addTimeoutObject(this);
+ }
+ }
- // TODO ZK: finish race.
+ /**
+ * @param topVer Topology version.
+ * @return {@code False} if future was already completed and need create another future instance.
+ */
+ boolean onStartResolveRequest(long topVer) {
+ synchronized (this) {
+ if (state == State.DONE)
+ return false;
+
+ if (state == State.WAIT_TIMEOUT)
+ impl.spi.getSpiContext().removeTimeoutObject(this);
+
+ assert resolveTopVer == 0 : resolveTopVer;
+
+ resolveTopVer = topVer;
+
+ state = State.RESOLVE_STARTED;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param node Node.
+ * @return Future finished when communication error resolve is done or {@code null} if another
+ * resolve process should be started.
+ */
+ @Nullable IgniteInternalFuture<Boolean> nodeStatusFuture(ClusterNode node) {
+ GridFutureAdapter<Boolean> fut;
synchronized (this) {
- fut = errNodes.get(nodeId);
+ if (state == State.DONE) {
+ if (resolveTopVer != 0 && node.order() <= resolveTopVer) {
+ Boolean res = !F.contains(resFailedNodes, node.order());
+
+ return new GridFinishedFuture<>(res);
+ }
+ else
+ return null;
+ }
+
+ fut = nodeFuts.get(node.order());
if (fut == null)
- errNodes.put(nodeId, fut = new GridFutureAdapter<>());
+ nodeFuts.put(node.order(), fut = new GridFutureAdapter<>());
}
- if (impl.node(nodeId) == null)
+ if (impl.node(node.order()) == null)
fut.onDone(false);
return fut;
}
/**
- * @param nodeId Node ID.
+ * @param node Failed node.
*/
- void onNodeFailed(UUID nodeId) {
- GridFutureAdapter<Boolean> fut;
+ void onNodeFailed(ClusterNode node) {
+ GridFutureAdapter<Boolean> fut = null;
synchronized (this) {
- fut = errNodes.get(nodeId);
+ if (state == State.WAIT_TIMEOUT)
+ fut = nodeFuts.get(node.order());
}
if (fut != null)
@@ -92,14 +193,50 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
/** {@inheritDoc} */
@Override public void run() {
- if (checkNotDoneOnTimeout()) {
+ // Run from zk discovery worker pool after timeout.
+ if (processTimeout()) {
try {
- impl.sendCustomMessage(new ZkInternalCommunicationErrorMessage());
+ impl.sendCustomMessage(new ZkCommunicationErrorResolveStartMessage(UUID.randomUUID()));
}
catch (Exception e) {
- onError(e);
+ Collection<GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ if (state != State.WAIT_TIMEOUT)
+ return;
+
+ state = State.DONE;
+
+ futs = nodeFuts.values(); // nodeFuts should not be modified after state changed to DONE.
+ }
+
+ for (GridFutureAdapter<Boolean> fut : futs)
+ fut.onDone(e);
+
+ onDone(e);
+ }
+ }
+ }
+
+ /**
+ * @return {@code True} if need initiate resolve process after timeout expired.
+ */
+ private boolean processTimeout() {
+ synchronized (this) {
+ if (state != State.WAIT_TIMEOUT)
+ return false;
+
+ for (GridFutureAdapter<Boolean> fut : nodeFuts.values()) {
+ if (!fut.isDone())
+ return true;
}
+
+ state = State.DONE;
}
+
+ onDone(null, null);
+
+ return false;
}
/** {@inheritDoc} */
@@ -114,43 +251,37 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
/** {@inheritDoc} */
@Override public void onTimeout() {
- if (isDone())
- return;
-
- if (checkNotDoneOnTimeout())
+ if (processTimeout())
impl.runInWorkerThread(this);
}
- /**
- * @param e Error.
- */
- private void onError(Exception e) {
- List<GridFutureAdapter<Boolean>> futs;
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
+ if (super.onDone(res, err)) {
+ impl.clearCommunicationErrorProcessFuture(this);
- synchronized (this) {
- futs = new ArrayList<>(errNodes.values());
+ return true;
}
- for (GridFutureAdapter<Boolean> fut : futs)
- fut.onDone(e);
+ return false;
+ }
- onDone(e);
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkCommunicationErrorProcessFuture.class, this);
}
/**
- * @return {@code True} if future already finished.
+ *
*/
- private boolean checkNotDoneOnTimeout() {
- // TODO ZK check state.
- synchronized (this) {
- for (GridFutureAdapter<Boolean> fut : errNodes.values()) {
- if (!fut.isDone())
- return false;
- }
- }
+ enum State {
+ /** */
+ DONE,
- onDone(null);
+ /** */
+ WAIT_TIMEOUT,
- return true;
+ /** */
+ RESOLVE_STARTED
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
new file mode 100644
index 0000000..144a5bf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+
+/**
+ *
+ */
+class ZkCommunicationErrorResolveFinishMessage implements ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final UUID futId;
+
+ /**
+ * @param futId Future ID.
+ */
+ ZkCommunicationErrorResolveFinishMessage(UUID futId) {
+ this.futId = futId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
new file mode 100644
index 0000000..e619d7b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class ZkCommunicationErrorResolveStartMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final UUID id;
+
+ /**
+ * @param id Unique ID.
+ */
+ ZkCommunicationErrorResolveStartMessage(UUID id) {
+ this.id = id;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
index 37dc7df..faea49e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
@@ -20,6 +20,7 @@ package org.apache.ignite.spi.discovery.zk.internal;
import java.io.Serializable;
import java.util.Collection;
import java.util.TreeMap;
+import java.util.UUID;
import org.jetbrains.annotations.Nullable;
/**
@@ -44,6 +45,9 @@ class ZkDiscoveryEventsData implements Serializable {
/** */
TreeMap<Long, ZkDiscoveryEventData> evts;
+ /** */
+ private UUID commErrFutId;
+
/**
* @param topVer Current topology version.
* @param gridStartTime Cluster start time.
@@ -56,6 +60,20 @@ class ZkDiscoveryEventsData implements Serializable {
}
/**
+ * @return Future ID.
+ */
+ @Nullable UUID communicationErrorResolveFutureId() {
+ return commErrFutId;
+ }
+
+ /**
+ * @param id Future ID.
+ */
+ void communicationErrorResolveFutureId(UUID id) {
+ commErrFutId = id;
+ }
+
+ /**
* @param nodes Current nodes in topology (these nodes should ack that event processed).
* @param evt Event.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkEventAckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkEventAckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkEventAckFuture.java
deleted file mode 100644
index ffe65c3..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkEventAckFuture.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.zk.internal;
-
-import java.util.List;
-import java.util.Set;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.zookeeper.AsyncCallback;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public class ZkEventAckFuture extends GridFutureAdapter<Void> implements Watcher, AsyncCallback.Children2Callback {
- /** */
- private final IgniteLogger log;
-
- /** */
- private final ZookeeperDiscoveryImpl impl;
-
- /** */
- private final Long evtId;
-
- /** */
- private final String evtPath;
-
- /** */
- private final int expAcks;
-
- /** */
- private final Set<Integer> remaininAcks;
-
- /**
- * @param impl
- * @param evtPath
- * @param evtId
- */
- ZkEventAckFuture(ZookeeperDiscoveryImpl impl, String evtPath, Long evtId) {
- this.impl = impl;
- this.log = impl.log();
- this.evtPath = evtPath;
- this.evtId = evtId;
-
- ZkClusterNodes top = impl.nodes();
-
- remaininAcks = U.newHashSet(top.nodesById.size());
-
- for (ZookeeperClusterNode node : top.nodesByInternalId.values()) {
- if (!node.isLocal())
- remaininAcks.add(node.internalId());
- }
-
- expAcks = remaininAcks.size();
-
- if (expAcks == 0)
- onDone();
- else
- impl.zkClient().getChildrenAsync(evtPath, this, this);
- }
-
- /**
- * @return Event ID.
- */
- Long eventId() {
- return evtId;
- }
-
- /**
- * @param node Failed node.
- */
- void onNodeFail(ZookeeperClusterNode node) {
- assert !remaininAcks.isEmpty();
-
- if (remaininAcks.remove(node.internalId()) && remaininAcks.isEmpty())
- onDone();
- }
-
- /** {@inheritDoc} */
- @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
- if (super.onDone(res, err)) {
- return true;
- }
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void process(WatchedEvent evt) {
- if (isDone())
- return;
-
- if (evt.getType() == Event.EventType.NodeChildrenChanged) {
- if (evtPath.equals(evt.getPath()))
- impl.zkClient().getChildrenAsync(evtPath, this, this);
- else
- U.warn(log, "Received event for unknown path: " + evt.getPath());
- }
- }
-
- /** {@inheritDoc} */
- @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
- assert rc == 0 : KeeperException.Code.get(rc);
-
- if (isDone())
- return;
-
- if (expAcks == stat.getCversion()) {
- log.info("Received expected number of acks [expCnt=" + expAcks + ", cVer=" + stat.getCversion() + ']');
-
- onDone();
- }
- else {
- for (int i = 0; i < children.size(); i++) {
- Integer nodeInternalId = Integer.parseInt(children.get(i));
-
- if (remaininAcks.remove(nodeInternalId) && remaininAcks.size() == 0)
- onDone();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
new file mode 100644
index 0000000..333f457
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class ZkForceNodeFailMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final UUID nodeId;
+
+ /** */
+ final String warning;
+
+ /**
+ * @param nodeId Node ID.
+ * @param warning Warning to be displayed on all nodes.
+ */
+ ZkForceNodeFailMessage(UUID nodeId, String warning) {
+ this.nodeId = nodeId;
+ this.warning = warning;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
index f08032a..06c5d9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
@@ -292,4 +292,12 @@ class ZkIgnitePaths {
String ackEventDataPath(long evtId) {
return customEvtsAcksDir + "/" + String.valueOf(evtId);
}
+
+ /**
+ * @param id Future ID.
+ * @return Future path.
+ */
+ String distributedFutureBasePath(UUID id) {
+ return evtsPath + "/f-" + id;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalCommunicationErrorMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalCommunicationErrorMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalCommunicationErrorMessage.java
deleted file mode 100644
index d7ed7ab..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalCommunicationErrorMessage.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.zk.internal;
-
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public class ZkInternalCommunicationErrorMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isMutable() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalForceNodeFailMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalForceNodeFailMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalForceNodeFailMessage.java
deleted file mode 100644
index 8d7a3df..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalForceNodeFailMessage.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.zk.internal;
-
-import java.util.UUID;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public class ZkInternalForceNodeFailMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- final UUID nodeId;
-
- /** */
- final String warning;
-
- /**
- * @param nodeId Node ID.
- * @param warning Warning to be displayed on all nodes.
- */
- ZkInternalForceNodeFailMessage(UUID nodeId, String warning) {
- this.nodeId = nodeId;
- this.warning = warning;
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isMutable() {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
new file mode 100644
index 0000000..fb6cf89
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+public abstract class ZkRunnable extends ZkAbstractCallabck implements Runnable {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ ZkRunnable(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ if (!onProcessStart())
+ return;
+
+ try {
+ run0();
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+
+ /**
+ *
+ */
+ protected abstract void run0() throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/376a4845/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index d21c18b..a153d11 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -28,6 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -43,6 +44,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.EventType;
import org.apache.ignite.internal.ClusterMetricsSnapshot;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
@@ -52,6 +54,7 @@ import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.MarshallerUtils;
@@ -88,7 +91,7 @@ public class ZookeeperDiscoveryImpl {
static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD";
/** */
- private final ZookeeperDiscoverySpi spi;
+ final ZookeeperDiscoverySpi spi;
/** */
private final String igniteInstanceName;
@@ -109,7 +112,7 @@ public class ZookeeperDiscoveryImpl {
private final IgniteLogger log;
/** */
- private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
+ final GridSpinBusyLock busyLock = new GridSpinBusyLock();
/** */
private final ZookeeperClusterNode locNode;
@@ -217,9 +220,25 @@ public class ZookeeperDiscoveryImpl {
return rtState.top.nodesById.get(nodeId);
}
+ /**
+ * @param nodeOrder Node order.
+ * @return Node instance.
+ */
+ @Nullable public ZookeeperClusterNode node(long nodeOrder) {
+ assert nodeOrder > 0 : nodeOrder;
+
+ return rtState.top.nodesByOrder.get(nodeOrder);
+ }
+
/** */
private final AtomicReference<ZkCommunicationErrorProcessFuture> commErrProcFut = new AtomicReference<>();
+ void clearCommunicationErrorProcessFuture(ZkCommunicationErrorProcessFuture fut) {
+ assert fut.isDone() : fut;
+
+ commErrProcFut.compareAndSet(fut, null);
+ }
+
/**
* @param node0 Problem node ID
* @param err Connect error.
@@ -230,24 +249,41 @@ public class ZookeeperDiscoveryImpl {
if (node == null)
return;
- ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+ IgniteInternalFuture<Boolean> nodeStatusFut;
+
+ for (;;) {
+ ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+ if (fut == null || fut.isDone()) {
+ ZkCommunicationErrorProcessFuture newFut = ZkCommunicationErrorProcessFuture.createOnCommunicationError(
+ this,
+ node.sessionTimeout() + 1000);
+
+ if (commErrProcFut.compareAndSet(fut, newFut)) {
+ fut = newFut;
- if (fut == null || fut.isDone()) {
- ZkCommunicationErrorProcessFuture newFut = new ZkCommunicationErrorProcessFuture(
- this,
- node.sessionTimeout() + 1000);
+ fut.scheduleCheckOnTimeout();
+ }
+ else
+ fut = commErrProcFut.get();
+ }
- if (commErrProcFut.compareAndSet(fut, newFut)) {
- fut = newFut;
+ nodeStatusFut = fut.nodeStatusFuture(node);
- spi.getSpiContext().addTimeoutObject(fut);
+ if (nodeStatusFut != null)
+ break;
+ else {
+ try {
+ fut.get();
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Previous communication error process future failed: " + e);
+ }
}
- else
- fut = commErrProcFut.get();
}
try {
- fut.nodeStatusFuture(node.id()).get();
+ nodeStatusFut.get();
}
catch (IgniteCheckedException e) {
throw new IgniteSpiException(e);
@@ -312,7 +348,7 @@ public class ZookeeperDiscoveryImpl {
return;
}
- sendCustomMessage(new ZkInternalForceNodeFailMessage(nodeId, warning));
+ sendCustomMessage(new ZkForceNodeFailMessage(nodeId, warning));
}
/**
@@ -1503,22 +1539,23 @@ public class ZookeeperDiscoveryImpl {
}
/**
+ * @param zkClient Client.
* @param evtPath Event path.
* @param sndNodeId Sender node ID.
* @return Event data.
* @throws Exception If failed.
*/
- private byte[] readCustomEventData(String evtPath, UUID sndNodeId) throws Exception {
+ private byte[] readCustomEventData(ZookeeperClient zkClient, String evtPath, UUID sndNodeId) throws Exception {
int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
if (partCnt > 1) {
String partsBasePath = zkPaths.customEventPartsBasePath(
ZkIgnitePaths.customEventPrefix(evtPath), sndNodeId);
- return readMultipleParts(rtState.zkClient, partsBasePath, partCnt);
+ return readMultipleParts(zkClient, partsBasePath, partCnt);
}
else
- return rtState.zkClient.getData(zkPaths.customEvtsDir + "/" + evtPath);
+ return zkClient.getData(zkPaths.customEvtsDir + "/" + evtPath);
}
/**
@@ -1528,6 +1565,9 @@ public class ZookeeperDiscoveryImpl {
private void generateCustomEvents(List<String> customEvtNodes) throws Exception {
assert rtState.crd;
+ ZookeeperClient zkClient = rtState.zkClient;
+ ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
TreeMap<Integer, String> newEvts = null;
for (int i = 0; i < customEvtNodes.size(); i++) {
@@ -1535,7 +1575,7 @@ public class ZookeeperDiscoveryImpl {
int evtSeq = ZkIgnitePaths.customEventSequence(evtPath);
- if (evtSeq > rtState.evtsData.procCustEvt) {
+ if (evtSeq > evtsData.procCustEvt) {
if (newEvts == null)
newEvts = new TreeMap<>();
@@ -1547,6 +1587,8 @@ public class ZookeeperDiscoveryImpl {
Set<UUID> alives = null;
for (Map.Entry<Integer, String> evtE : newEvts.entrySet()) {
+ evtsData.procCustEvt = evtE.getKey();
+
String evtPath = evtE.getValue();
UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
@@ -1557,23 +1599,21 @@ public class ZookeeperDiscoveryImpl {
sndNode = null;
if (sndNode != null) {
- byte[] evtBytes = readCustomEventData(evtPath, sndNodeId);
+ byte[] evtBytes = readCustomEventData(zkClient, evtPath, sndNodeId);
DiscoverySpiCustomMessage msg;
try {
msg = unmarshalZip(evtBytes);
- rtState.evtsData.evtIdGen++;
-
- if (msg instanceof ZkInternalForceNodeFailMessage) {
- ZkInternalForceNodeFailMessage msg0 = (ZkInternalForceNodeFailMessage)msg;
+ if (msg instanceof ZkForceNodeFailMessage) {
+ ZkForceNodeFailMessage msg0 = (ZkForceNodeFailMessage)msg;
if (alives == null)
alives = new HashSet<>(rtState.top.nodesById.keySet());
if (alives.contains(msg0.nodeId)) {
- rtState.evtsData.topVer++;
+ evtsData.topVer++;
alives.remove(msg0.nodeId);
@@ -1581,9 +1621,9 @@ public class ZookeeperDiscoveryImpl {
assert node != null : msg0.nodeId;
- for (String child : zkClient().getChildren(zkPaths.aliveNodesDir)) {
+ for (String child : zkClient.getChildren(zkPaths.aliveNodesDir)) {
if (ZkIgnitePaths.aliveInternalId(child) == node.internalId()) {
- zkClient().deleteIfExistsAsync(zkPaths.aliveNodesDir + "/" + child);
+ zkClient.deleteIfExistsAsync(zkPaths.aliveNodesDir + "/" + child);
break;
}
@@ -1593,20 +1633,51 @@ public class ZookeeperDiscoveryImpl {
if (log.isDebugEnabled())
log.debug("Ignore forcible node fail request for unknown node: " + msg0.nodeId);
+ deleteCustomEventDataAsync(zkClient, evtPath);
+
+ continue;
+ }
+ }
+ else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
+ ZkCommunicationErrorResolveStartMessage msg0 =
+ (ZkCommunicationErrorResolveStartMessage)msg;
+
+ if (evtsData.communicationErrorResolveFutureId() != null) {
+ if (log.isInfoEnabled()) {
+ log.info("Ignore communication error resolve message, resolve process " +
+ "already started [sndNode=" + sndNode + ']');
+ }
+
+ deleteCustomEventDataAsync(zkClient, evtPath);
+
continue;
}
+ else {
+ if (log.isInfoEnabled()) {
+ log.info("Start communication error resolve [sndNode=" + sndNode +
+ ", topVer=" + evtsData.topVer + ']');
+ }
+
+ zkClient.createIfNeeded(zkPaths.distributedFutureBasePath(msg0.id),
+ null,
+ PERSISTENT);
+
+ evtsData.communicationErrorResolveFutureId(msg0.id);
+ }
}
+ evtsData.evtIdGen++;
+
ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
- rtState.evtsData.evtIdGen,
- rtState.evtsData.topVer,
+ evtsData.evtIdGen,
+ evtsData.topVer,
sndNodeId,
evtPath,
false);
evtData.msg = msg;
- rtState.evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
+ evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
if (log.isDebugEnabled())
log.debug("Generated CUSTOM event [evt=" + evtData + ", msg=" + msg + ']');
@@ -1622,8 +1693,6 @@ public class ZookeeperDiscoveryImpl {
deleteCustomEventDataAsync(rtState.zkClient, evtPath);
}
-
- rtState.evtsData.procCustEvt = evtE.getKey();
}
saveAndProcessNewEvents();
@@ -1695,6 +1764,8 @@ public class ZookeeperDiscoveryImpl {
private void processNewEvents(final ZkDiscoveryEventsData evtsData) throws Exception {
TreeMap<Long, ZkDiscoveryEventData> evts = evtsData.evts;
+ ZookeeperClient zkClient = rtState.zkClient;
+
boolean updateNodeInfo = false;
for (ZkDiscoveryEventData evtData : evts.tailMap(rtState.locNodeInfo.lastProcEvt, false).values()) {
@@ -1768,10 +1839,12 @@ public class ZookeeperDiscoveryImpl {
if (evtData0.ackEvent()) {
String path = zkPaths.ackEventDataPath(evtData0.eventId());
- msg = unmarshalZip(rtState.zkClient.getData(path));
+ msg = unmarshalZip(zkClient.getData(path));
}
else {
- byte[] msgBytes = readCustomEventData(evtData0.evtPath, evtData0.sndNodeId);
+ byte[] msgBytes = readCustomEventData(zkClient,
+ evtData0.evtPath,
+ evtData0.sndNodeId);
msg = unmarshalZip(msgBytes);
}
@@ -1814,7 +1887,7 @@ public class ZookeeperDiscoveryImpl {
if (log.isDebugEnabled())
log.debug("Update processed events: " + rtState.locNodeInfo.lastProcEvt);
- rtState.zkClient.setData(rtState.locNodeZkPath, marshalZip(rtState.locNodeInfo), -1);
+ zkClient.setData(rtState.locNodeZkPath, marshalZip(rtState.locNodeInfo), -1);
}
}
@@ -1897,29 +1970,134 @@ public class ZookeeperDiscoveryImpl {
* @throws Exception If failed.
*/
private void processInternalMessage(ZkDiscoveryCustomEventData evtData, ZkInternalMessage msg) throws Exception {
- if (msg instanceof ZkInternalForceNodeFailMessage) {
- ZkInternalForceNodeFailMessage msg0 = (ZkInternalForceNodeFailMessage)msg;
+ if (msg instanceof ZkForceNodeFailMessage)
+ processForceNodeFailMessage((ZkForceNodeFailMessage)msg, evtData);
+ else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
+ processStartResolveCommunicationErrorMessage(
+ (ZkCommunicationErrorResolveStartMessage)msg,
+ evtData);
+ }
+ else if (msg instanceof ZkCommunicationErrorResolveFinishMessage) {
+ ZkCommunicationErrorResolveFinishMessage msg0 = (ZkCommunicationErrorResolveFinishMessage)msg;
+ }
+ }
+
+ /**
+ * @param msg Message.
+ * @param evtData Event data.
+ * @throws Exception If failed.
+ */
+ private void processForceNodeFailMessage(ZkForceNodeFailMessage msg, ZkDiscoveryCustomEventData evtData)
+ throws Exception {
+ ClusterNode creatorNode = rtState.top.nodesById.get(evtData.sndNodeId);
+
+ if (msg.warning != null) {
+ U.warn(log, "Received EVT_NODE_FAILED event with warning [" +
+ "nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) +
+ ", nodeId=" + msg.nodeId +
+ ", msg=" + msg.warning + ']');
+ }
+ else {
+ U.warn(log, "Received force EVT_NODE_FAILED event [" +
+ "nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) +
+ ", nodeId=" + msg.nodeId + ']');
+ }
+
+ ZookeeperClusterNode node = rtState.top.nodesById.get(msg.nodeId);
+
+ assert node != null : msg.nodeId;
+
+ processNodeFail(node.internalId(), evtData.topologyVersion());
+ }
+
+ /**
+ * @param msg Message.
+ * @param evtData Event data.
+ */
+ private void processStartResolveCommunicationErrorMessage(ZkCommunicationErrorResolveStartMessage msg,
+ ZkDiscoveryCustomEventData evtData) throws Exception {
+ ZkCommunicationErrorProcessFuture fut;
- ClusterNode creatorNode = rtState.top.nodesById.get(evtData.sndNodeId);
+ for (;;) {
+ fut = commErrProcFut.get();
+
+ if (fut == null || fut.isDone()) {
+ ZkCommunicationErrorProcessFuture newFut =
+ ZkCommunicationErrorProcessFuture.createOnStartResolveRequest(this);
- if (msg0.warning != null) {
- U.warn(log, "Received EVT_NODE_FAILED event with warning [" +
- "nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) +
- ", nodeId=" + msg0.nodeId +
- ", msg=" + msg0.warning + ']');
+ if (commErrProcFut.compareAndSet(fut, newFut))
+ fut = newFut;
+ else
+ fut = commErrProcFut.get();
}
+
+ if (fut.onStartResolveRequest(evtData.topologyVersion()))
+ break;
else {
- U.warn(log, "Received force EVT_NODE_FAILED event [" +
- "nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) +
- ", nodeId=" + msg0.nodeId + ']');
+ try {
+ fut.get();
+ }
+ catch (Exception e) {
+ U.warn(log, "Previous communication error process future failed: " + e);
+ }
}
+ }
+
+ assert !fut.isDone() : fut;
- ZookeeperClusterNode node = rtState.top.nodesById.get(msg0.nodeId);
+ final String futPath = zkPaths.distributedFutureBasePath(msg.id);
+ final ZkCommunicationErrorProcessFuture fut0 = fut;
+ final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+ if (rtState.crd) {
+ ZkCollectDistributedFuture nodeResFut = new ZkCollectDistributedFuture(this, rtState, futPath,
+ new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ // Future is completed from ZK event thread.
+ finishCommunicationResolveProcess(rtState);
- assert node != null : msg0.nodeId;
+ return null;
+ }
+ }
+ );
- processNodeFail(node.internalId(), evtData.topologyVersion());
+ fut.nodeResultCollectFuture(nodeResFut);
}
+
+ runInWorkerThread(new ZkRunnable(rtState, this) {
+ @Override protected void run0() throws Exception {
+ fut0.pingNodesAndNotifyFuture(locNode.order(), rtState, futPath, topSnapshot);
+ }
+ });
+ }
+
+ /**
+ * @param rtState Runtime state.
+ * @throws Exception If failed.
+ */
+ void finishCommunicationResolveProcess(ZkRuntimeState rtState) throws Exception {
+ ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
+ UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
+
+ assert futId != null;
+
+ rtState.evtsData.communicationErrorResolveFutureId(null);
+
+ ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId);
+
+ ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
+ evtsData.evtIdGen,
+ evtsData.topVer,
+ locNode.id(),
+ null,
+ false);
+
+ evtData.msg = msg;
+
+ evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
+
+ saveAndProcessNewEvents();
}
/**
@@ -2041,7 +2219,7 @@ public class ZookeeperDiscoveryImpl {
ZkCommunicationErrorProcessFuture commErrFut = commErrProcFut.get();
if (commErrFut != null)
- commErrFut.onNodeFailed(failedNode.id());
+ commErrFut.onNodeFailed(failedNode);
final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
@@ -2301,8 +2479,6 @@ public class ZookeeperDiscoveryImpl {
connState = ConnectionState.STOPPED;
}
- ZookeeperClient zkClient = rtState.zkClient;
-
rtState.onCloseStart();
busyLock.block();
@@ -2311,6 +2487,8 @@ public class ZookeeperDiscoveryImpl {
joinFut.onDone(e);
+ ZookeeperClient zkClient = rtState.zkClient;
+
if (zkClient != null)
zkClient.close();
@@ -2321,7 +2499,7 @@ public class ZookeeperDiscoveryImpl {
* @param busyLock Busy lock.
* @param err Error.
*/
- private void onFatalError(GridSpinBusyLock busyLock, Throwable err) {
+ void onFatalError(GridSpinBusyLock busyLock, Throwable err) {
busyLock.leaveBusy();
if (err instanceof ZookeeperClientFailedException)
@@ -2444,111 +2622,12 @@ public class ZookeeperDiscoveryImpl {
/**
*
*/
- abstract class ZkCallabck {
- /** */
- final ZkRuntimeState rtState;
-
- /**
- * @param rtState Runtime state.
- */
- ZkCallabck(ZkRuntimeState rtState) {
- this.rtState = rtState;
- }
-
- /**
- * @return {@code True} if is able to start processing.
- */
- final boolean onProcessStart() {
- return !rtState.closing && busyLock.enterBusy();
- }
-
- /**
- *
- */
- final void onProcessEnd() {
- busyLock.leaveBusy();
- }
-
- /**
- * @param e Error.
- */
- final void onProcessError(Throwable e) {
- onFatalError(busyLock, e);
- }
- }
-
- /**
- *
- */
- abstract class AbstractWatcher extends ZkCallabck implements Watcher {
- /**
- * @param rtState Runtime state.
- */
- AbstractWatcher(ZkRuntimeState rtState) {
- super(rtState);
- }
-
- /** {@inheritDoc} */
- @Override public final void process(WatchedEvent evt) {
- if (!onProcessStart())
- return;
-
- try {
- process0(evt);
-
- onProcessEnd();
- }
- catch (Throwable e) {
- onProcessError(e);
- }
- }
-
- /**
- * @param evt Event.
- * @throws Exception If failed.
- */
- protected abstract void process0(WatchedEvent evt) throws Exception;
- }
-
- /**
- *
- */
- abstract class AbstractChildrenCallback extends ZkCallabck implements AsyncCallback.Children2Callback {
- /**
- * @param rtState Runtime state.
- */
- AbstractChildrenCallback(ZkRuntimeState rtState) {
- super(rtState);
- }
-
- /** {@inheritDoc} */
- @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
- if (!onProcessStart())
- return;
-
- try {
- processResult0(rc, path, ctx, children, stat);
-
- onProcessEnd();
- }
- catch (Throwable e) {
- onProcessError(e);
- }
- }
-
- abstract void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
- throws Exception;
- }
-
- /**
- *
- */
- private class ZkWatcher extends AbstractWatcher implements ZkRuntimeState.ZkWatcher {
+ private class ZkWatcher extends ZkAbstractWatcher implements ZkRuntimeState.ZkWatcher {
/**
* @param rtState Runtime state.
*/
ZkWatcher(ZkRuntimeState rtState) {
- super(rtState);
+ super(rtState, ZookeeperDiscoveryImpl.this);
}
/** {@inheritDoc} */
@@ -2619,12 +2698,12 @@ public class ZookeeperDiscoveryImpl {
/**
*
*/
- private class AliveNodeDataWatcher extends AbstractWatcher implements ZkRuntimeState.ZkAliveNodeDataWatcher {
+ private class AliveNodeDataWatcher extends ZkAbstractWatcher implements ZkRuntimeState.ZkAliveNodeDataWatcher {
/**
* @param rtState Runtime state.
*/
AliveNodeDataWatcher(ZkRuntimeState rtState) {
- super(rtState);
+ super(rtState, ZookeeperDiscoveryImpl.this);
}
/** {@inheritDoc} */
@@ -2691,12 +2770,12 @@ public class ZookeeperDiscoveryImpl {
/**
*
*/
- private class PreviousNodeWatcher extends AbstractWatcher implements AsyncCallback.StatCallback {
+ private class PreviousNodeWatcher extends ZkAbstractWatcher implements AsyncCallback.StatCallback {
/**
* @param rtState Runtime state.
*/
PreviousNodeWatcher(ZkRuntimeState rtState) {
- super(rtState);
+ super(rtState, ZookeeperDiscoveryImpl.this);
}
/** {@inheritDoc} */
@@ -2731,12 +2810,12 @@ public class ZookeeperDiscoveryImpl {
/**
*
*/
- class CheckCoordinatorCallback extends AbstractChildrenCallback {
+ class CheckCoordinatorCallback extends ZkAbstractChildrenCallback {
/**
* @param rtState Runtime state.
*/
CheckCoordinatorCallback(ZkRuntimeState rtState) {
- super(rtState);
+ super(rtState, ZookeeperDiscoveryImpl.this);
}
/** {@inheritDoc} */
[40/50] [abbrv] ignite git commit: IGNITE-7232 .NET: Cross-platform
examples: remove localhost from config to join nodes with default config
Posted by sb...@apache.org.
IGNITE-7232 .NET: Cross-platform examples: remove localhost from config to join nodes with default config
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a2b195ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a2b195ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a2b195ce
Branch: refs/heads/ignite-zk
Commit: a2b195ced34af644117fde7cbe43f1fd2cbf9db3
Parents: 5cd3e26
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Mon Dec 18 18:31:25 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Mon Dec 18 18:31:25 2017 +0300
----------------------------------------------------------------------
modules/platforms/dotnet/examples/dotnetcore/App.config | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a2b195ce/modules/platforms/dotnet/examples/dotnetcore/App.config
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/dotnetcore/App.config b/modules/platforms/dotnet/examples/dotnetcore/App.config
index e653d62..32c8b31 100644
--- a/modules/platforms/dotnet/examples/dotnetcore/App.config
+++ b/modules/platforms/dotnet/examples/dotnetcore/App.config
@@ -26,10 +26,7 @@
<gcServer enabled="true" />
</runtime>
- <igniteConfiguration xmlns="http://ignite.apache.org/schema/dotnet/IgniteConfigurationSection"
- localhost="127.0.0.1">
- <atomicConfiguration atomicSequenceReserveSize="10" />
-
+ <igniteConfiguration xmlns="http://ignite.apache.org/schema/dotnet/IgniteConfigurationSection">
<discoverySpi type="TcpDiscoverySpi">
<ipFinder type="TcpDiscoveryMulticastIpFinder">
<endpoints>
[10/50] [abbrv] ignite git commit: IGNITE-6995 Visor CMD: Updated
eviction policy factory in configs.
Posted by sb...@apache.org.
IGNITE-6995 Visor CMD: Updated eviction policy factory in configs.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/47e7daf7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/47e7daf7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/47e7daf7
Branch: refs/heads/ignite-zk
Commit: 47e7daf7323ee49be53efb363f447d574bc7e222
Parents: 9f7bc54
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Dec 13 23:33:05 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Dec 13 23:33:05 2017 +0700
----------------------------------------------------------------------
.../cache/VisorCacheEvictionConfiguration.java | 4 +-
.../cache/VisorCacheNearConfiguration.java | 8 ++-
.../internal/visor/util/VisorTaskUtils.java | 14 ++---
.../generator/AbstractTransformer.js | 8 +--
.../generator/ConfigurationGenerator.js | 66 ++++++++++++--------
.../generator/PlatformGenerator.js | 6 +-
.../generator/defaults/Cache.service.js | 14 +----
.../frontend/app/services/Version.service.js | 4 ++
8 files changed, 66 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheEvictionConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheEvictionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheEvictionConfiguration.java
index 7792d8e..496c609 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheEvictionConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheEvictionConfiguration.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.visor.cache;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
-import org.apache.ignite.cache.eviction.EvictionPolicy;
+import javax.cache.configuration.Factory;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -58,7 +58,7 @@ public class VisorCacheEvictionConfiguration extends VisorDataTransferObject {
* @param ccfg Cache configuration.
*/
public VisorCacheEvictionConfiguration(CacheConfiguration ccfg) {
- final EvictionPolicy evictionPlc = ccfg.getEvictionPolicy();
+ final Factory evictionPlc = ccfg.getEvictionPolicyFactory();
plc = compactClass(evictionPlc);
plcMaxSize = evictionPolicyMaxSize(evictionPlc);
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNearConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNearConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNearConfiguration.java
index a4cd3d9..a7b431c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNearConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNearConfiguration.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.visor.cache;
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
-import org.apache.ignite.cache.eviction.EvictionPolicy;
+import javax.cache.configuration.Factory;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.internal.processors.cache.GridCacheUtils;
@@ -69,9 +69,11 @@ public class VisorCacheNearConfiguration extends VisorDataTransferObject {
if (nearEnabled) {
NearCacheConfiguration nccfg = ccfg.getNearConfiguration();
+ final Factory nearEvictionPlc = nccfg.getNearEvictionPolicyFactory();
+
nearStartSize = nccfg.getNearStartSize();
- nearEvictPlc = compactClass(nccfg.getNearEvictionPolicy());
- nearEvictMaxSize = evictionPolicyMaxSize(nccfg.getNearEvictionPolicy());
+ nearEvictPlc = compactClass(nearEvictionPlc);
+ nearEvictMaxSize = evictionPolicyMaxSize(nearEvictionPlc);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index fda801c..0da7a34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -50,13 +50,12 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import javax.cache.configuration.Factory;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteFileSystem;
import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.cache.eviction.EvictionPolicy;
-import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicyMBean;
-import org.apache.ignite.cache.eviction.lru.LruEvictionPolicyMBean;
+import org.apache.ignite.cache.eviction.AbstractEvictionPolicyFactory;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.processors.igfs.IgfsEx;
@@ -759,12 +758,9 @@ public class VisorTaskUtils {
* @param plc Eviction policy.
* @return Extracted max size.
*/
- public static Integer evictionPolicyMaxSize(@Nullable EvictionPolicy plc) {
- if (plc instanceof LruEvictionPolicyMBean)
- return ((LruEvictionPolicyMBean)plc).getMaxSize();
-
- if (plc instanceof FifoEvictionPolicyMBean)
- return ((FifoEvictionPolicyMBean)plc).getMaxSize();
+ public static Integer evictionPolicyMaxSize(@Nullable Factory plc) {
+ if (plc instanceof AbstractEvictionPolicyFactory)
+ return ((AbstractEvictionPolicyFactory) plc).getMaxSize();
return null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
index af799bd..339dbea 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/AbstractTransformer.js
@@ -284,13 +284,13 @@ export default class AbstractTransformer {
}
// Generate server near cache group.
- static cacheNearServer(cache) {
- return this.toSection(this.generator.cacheNearServer(cache));
+ static cacheNearServer(cache, available) {
+ return this.toSection(this.generator.cacheNearServer(cache, available));
}
// Generate client near cache group.
- static cacheNearClient(cache) {
- return this.toSection(this.generator.cacheNearClient(cache));
+ static cacheNearClient(cache, available) {
+ return this.toSection(this.generator.cacheNearClient(cache, available));
}
// Generate cache statistics group.
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 16202f8..0c40890 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1887,34 +1887,52 @@ export default class IgniteConfigurationGenerator {
* @returns {Object} Parent configuration.
* @private
*/
- static _evictionPolicy(ccfg, name, src, dflt) {
- let bean;
+ static _evictionPolicy(ccfg, available, near, src, dflt) {
+ let propName;
+ let beanProps;
- switch (_.get(src, 'kind')) {
- case 'LRU':
- bean = new Bean('org.apache.ignite.cache.eviction.lru.LruEvictionPolicy', 'evictionPlc',
- src.LRU, dflt.LRU);
+ if (available('2.4.0')) {
+ switch (_.get(src, 'kind')) {
+ case 'LRU': beanProps = {cls: 'org.apache.ignite.cache.eviction.lru.LruEvictionPolicyFactory', src: src.LRU };
+ break;
- break;
- case 'FIFO':
- bean = new Bean('org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy', 'evictionPlc',
- src.FIFO, dflt.FIFO);
+ case 'FIFO': beanProps = {cls: 'org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicyFactory', src: src.FIFO };
+ break;
- break;
- case 'SORTED':
- bean = new Bean('org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy', 'evictionPlc',
- src.SORTED, dflt.SORTED);
+ case 'SORTED': beanProps = {cls: 'org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicyFactory', src: src.SORTED };
+ break;
- break;
- default:
- return ccfg;
+ default:
+ return ccfg;
+ }
+
+ propName = (near ? 'nearEviction' : 'eviction') + 'PolicyFactory';
}
+ else {
+ switch (_.get(src, 'kind')) {
+ case 'LRU': beanProps = {cls: 'org.apache.ignite.cache.eviction.lru.LruEvictionPolicy', src: src.LRU };
+ break;
+
+ case 'FIFO': beanProps = {cls: 'org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy', src: src.FIFO };
+ break;
+
+ case 'SORTED': beanProps = {cls: 'org.apache.ignite.cache.eviction.sorted.SortedEvictionPolicy', src: src.SORTED };
+ break;
+
+ default:
+ return ccfg;
+ }
+
+ propName = (near ? 'nearEviction' : 'eviction') + 'Policy';
+ }
+
+ const bean = new Bean(beanProps.cls, 'evictionPlc', beanProps.src, dflt);
bean.intProperty('batchSize')
.intProperty('maxMemorySize')
.intProperty('maxSize');
- ccfg.beanProperty(name, bean);
+ ccfg.beanProperty(propName, bean);
return ccfg;
}
@@ -2013,7 +2031,7 @@ export default class IgniteConfigurationGenerator {
.emptyBeanProperty('evictionFilter');
}
- this._evictionPolicy(ccfg, 'evictionPolicy', cache.evictionPolicy, cacheDflts.evictionPolicy);
+ this._evictionPolicy(ccfg, available, false, cache.evictionPolicy, cacheDflts.evictionPolicy);
// Removed in ignite 2.0
if (available(['1.0.0', '2.0.0'])) {
@@ -2249,15 +2267,14 @@ export default class IgniteConfigurationGenerator {
}
// Generate server near cache group.
- static cacheNearServer(cache, ccfg = this.cacheConfigurationBean(cache)) {
+ static cacheNearServer(cache, available, ccfg = this.cacheConfigurationBean(cache)) {
if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && _.get(cache, 'nearConfiguration.enabled')) {
const bean = new Bean('org.apache.ignite.configuration.NearCacheConfiguration', 'nearConfiguration',
cache.nearConfiguration, cacheDflts.nearConfiguration);
bean.intProperty('nearStartSize');
- this._evictionPolicy(bean, 'nearEvictionPolicy',
- bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
+ this._evictionPolicy(bean, available, true, bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
ccfg.beanProperty('nearConfiguration', bean);
}
@@ -2266,7 +2283,7 @@ export default class IgniteConfigurationGenerator {
}
// Generate client near cache group.
- static cacheNearClient(cache, ccfg = this.cacheConfigurationBean(cache)) {
+ static cacheNearClient(cache, available, ccfg = this.cacheConfigurationBean(cache)) {
if (ccfg.valueOf('cacheMode') === 'PARTITIONED' && _.get(cache, 'clientNearConfiguration.enabled')) {
const bean = new Bean('org.apache.ignite.configuration.NearCacheConfiguration',
javaTypes.toJavaName('nearConfiguration', ccfg.valueOf('name')),
@@ -2274,8 +2291,7 @@ export default class IgniteConfigurationGenerator {
bean.intProperty('nearStartSize');
- this._evictionPolicy(bean, 'nearEvictionPolicy',
- bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
+ this._evictionPolicy(bean, available, true, bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
return bean;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
index 234c7ec..2f652d4 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
@@ -291,13 +291,13 @@ export default ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatfo
}
// Generate cache memory group.
- static cacheMemory(cache, ccfg = this.cacheConfigurationBean(cache)) {
+ static cacheMemory(cache, available, ccfg = this.cacheConfigurationBean(cache)) {
ccfg.enumProperty('memoryMode');
if (ccfg.valueOf('memoryMode') !== 'OFFHEAP_VALUES')
ccfg.intProperty('offHeapMaxMemory');
- // this._evictionPolicy(ccfg, 'evictionPolicy', cache.evictionPolicy, cacheDflts.evictionPolicy);
+ // this._evictionPolicy(ccfg, available, false, cache.evictionPolicy, cacheDflts.evictionPolicy);
ccfg.intProperty('startSize')
.boolProperty('swapEnabled', 'EnableSwap');
@@ -483,7 +483,7 @@ export default ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatfo
bean.intProperty('nearStartSize');
- this._evictionPolicy(bean, 'nearEvictionPolicy',
+ this._evictionPolicy(bean, true,
bean.valueOf('nearEvictionPolicy'), cacheDflts.evictionPolicy);
ccfg.beanProperty('nearConfiguration', bean);
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
index ad5b88a..3f87b8a 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/defaults/Cache.service.js
@@ -85,18 +85,8 @@ const DFLT_CACHE = {
nearStartSize: 375000
},
evictionPolicy: {
- LRU: {
- batchSize: 1,
- maxSize: 100000
- },
- FIFO: {
- batchSize: 1,
- maxSize: 100000
- },
- SORTED: {
- batchSize: 1,
- maxSize: 100000
- }
+ batchSize: 1,
+ maxSize: 100000
},
queryMetadata: 'Configuration',
queryDetailMetricsSize: 0,
http://git-wip-us.apache.org/repos/asf/ignite/blob/47e7daf7/modules/web-console/frontend/app/services/Version.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/services/Version.service.js b/modules/web-console/frontend/app/services/Version.service.js
index 8b67eb7..22d0732 100644
--- a/modules/web-console/frontend/app/services/Version.service.js
+++ b/modules/web-console/frontend/app/services/Version.service.js
@@ -77,6 +77,10 @@ export default class IgniteVersion {
this.supportedVersions = [
{
+ label: 'Ignite 2.4',
+ ignite: '2.4.0'
+ },
+ {
label: 'Ignite 2.3',
ignite: '2.3.0'
},
[02/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b7cbd4ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b7cbd4ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b7cbd4ce
Branch: refs/heads/ignite-zk
Commit: b7cbd4cefaa8b60c3626d4a24afbfbcb1f490f39
Parents: fe515ee
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 12 17:52:03 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 12 17:52:03 2017 +0300
----------------------------------------------------------------------
.../ZkCommunicationErrorProcessFuture.java | 86 +++++++++++++++++---
.../zk/internal/ZookeeperDiscoveryImpl.java | 11 ++-
2 files changed, 79 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7cbd4ce/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index 91ecaf7..2ea65e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -17,11 +17,14 @@
package org.apache.ignite.spi.discovery.zk.internal;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.UUID;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
-import org.jboss.netty.util.internal.ConcurrentHashMap;
/**
*
@@ -31,7 +34,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
private final ZookeeperDiscoveryImpl impl;
/** */
- private final ConcurrentHashMap<UUID, GridFutureAdapter<Boolean>> errNodes = new ConcurrentHashMap<>();
+ private final Map<UUID, GridFutureAdapter<Boolean>> errNodes = new HashMap<>();
/** */
private final long endTime;
@@ -41,6 +44,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
/**
* @param impl Discovery implementation.
+ * @param timeout Wait timeout before initiating communication errors resolve.
*/
ZkCommunicationErrorProcessFuture(ZookeeperDiscoveryImpl impl, long timeout) {
this.impl = impl;
@@ -50,14 +54,20 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
endTime = System.currentTimeMillis() + timeout;
}
+ /**
+ * @param nodeId Node ID.
+ * @return Future finished when communication error resolve is done.
+ */
GridFutureAdapter<Boolean> nodeStatusFuture(UUID nodeId) {
- GridFutureAdapter<Boolean> fut = errNodes.get(nodeId);
+ GridFutureAdapter<Boolean> fut;
- if (fut == null) {
- GridFutureAdapter<Boolean> old = errNodes.putIfAbsent(nodeId, fut = new GridFutureAdapter<>());
+ // TODO ZK: finish race.
- if (old != null)
- fut = old;
+ synchronized (this) {
+ fut = errNodes.get(nodeId);
+
+ if (fut == null)
+ errNodes.put(nodeId, fut = new GridFutureAdapter<>());
}
if (impl.node(nodeId) == null)
@@ -66,8 +76,15 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
return fut;
}
+ /**
+ * @param nodeId Node ID.
+ */
void onNodeFailed(UUID nodeId) {
- GridFutureAdapter<Boolean> fut = errNodes.get(nodeId);
+ GridFutureAdapter<Boolean> fut;
+
+ synchronized (this) {
+ fut = errNodes.get(nodeId);
+ }
if (fut != null)
fut.onDone(false);
@@ -75,20 +92,65 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter implements Ign
/** {@inheritDoc} */
@Override public void run() {
- // TODO ZK
+ if (checkNotDoneOnTimeout()) {
+ try {
+ impl.sendCustomMessage(new ZkInternalCommunicationErrorMessage());
+ }
+ catch (Exception e) {
+ onError(e);
+ }
+ }
}
/** {@inheritDoc} */
@Override public IgniteUuid id() {
- return null;
+ return id;
}
+ /** {@inheritDoc} */
@Override public long endTime() {
- return 0;
+ return endTime;
}
/** {@inheritDoc} */
@Override public void onTimeout() {
- // TODO ZK
+ if (isDone())
+ return;
+
+ if (checkNotDoneOnTimeout())
+ impl.runInWorkerThread(this);
+ }
+
+ /**
+ * @param e Error.
+ */
+ private void onError(Exception e) {
+ List<GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ futs = new ArrayList<>(errNodes.values());
+ }
+
+ for (GridFutureAdapter<Boolean> fut : futs)
+ fut.onDone(e);
+
+ onDone(e);
+ }
+
+ /**
+ * @return {@code True} if future already finished.
+ */
+ private boolean checkNotDoneOnTimeout() {
+ // TODO ZK check state.
+ synchronized (this) {
+ for (GridFutureAdapter<Boolean> fut : errNodes.values()) {
+ if (!fut.isDone())
+ return false;
+ }
+ }
+
+ onDone(null);
+
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/b7cbd4ce/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 783595f..d21c18b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -225,9 +225,6 @@ public class ZookeeperDiscoveryImpl {
* @param err Connect error.
*/
public void onCommunicationConnectionError(ClusterNode node0, Exception err) {
- if (true)
- return;
-
ZookeeperClusterNode node = node(node0.id());
if (node == null)
@@ -236,12 +233,14 @@ public class ZookeeperDiscoveryImpl {
ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
if (fut == null || fut.isDone()) {
- ZkCommunicationErrorProcessFuture newFut = new ZkCommunicationErrorProcessFuture(this, node.sessionTimeout());
+ ZkCommunicationErrorProcessFuture newFut = new ZkCommunicationErrorProcessFuture(
+ this,
+ node.sessionTimeout() + 1000);
if (commErrProcFut.compareAndSet(fut, newFut)) {
fut = newFut;
- sendCustomMessage(new ZkInternalCommunicationErrorMessage());
+ spi.getSpiContext().addTimeoutObject(fut);
}
else
fut = commErrProcFut.get();
@@ -2255,7 +2254,7 @@ public class ZookeeperDiscoveryImpl {
/**
* @param c Closure to run.
*/
- private void runInWorkerThread(Runnable c) {
+ void runInWorkerThread(Runnable c) {
IgniteThreadPoolExecutor pool;
synchronized (stateMux) {
[41/50] [abbrv] ignite git commit: IGNITE-7229 .NET: Rename
ICache.QueryFields to Query, add FieldNames
Posted by sb...@apache.org.
IGNITE-7229 .NET: Rename ICache.QueryFields to Query, add FieldNames
This closes #3247
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8f327a18
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8f327a18
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8f327a18
Branch: refs/heads/ignite-zk
Commit: 8f327a18bea8cd4efda0d9ffea1acf7a11d56f90
Parents: a2b195c
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Tue Dec 19 10:20:08 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Tue Dec 19 10:20:08 2017 +0300
----------------------------------------------------------------------
.../query/PlatformAbstractQueryCursor.java | 9 +++
.../cache/query/PlatformFieldsQueryCursor.java | 24 ++++++-
.../ExpiryCacheHolderTest.cs | 5 ++
.../Binary/BinaryFooterTest.cs | 2 +-
.../Binary/Serializable/SqlDmlTest.cs | 6 +-
.../Cache/CacheTestAsyncWrapper.cs | 7 ++
.../Cache/Query/CacheDmlQueriesTest.cs | 18 ++---
.../Query/CacheQueriesCodeConfigurationTest.cs | 2 +-
.../Cache/Query/CacheQueriesTest.cs | 69 +++++++++++++++-----
.../Query/Linq/CacheLinqTest.Introspection.cs | 2 +-
.../Client/Cache/SqlQueryTest.cs | 4 +-
.../dotnet/Apache.Ignite.Core/Cache/ICache.cs | 8 +++
.../Impl/Binary/BinaryReaderExtensions.cs | 2 +-
.../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 29 +++++---
.../Impl/Cache/ICacheInternal.cs | 2 +-
.../Impl/Cache/Query/FieldsQueryCursor.cs | 39 +++++++++++
.../Impl/Cache/Query/PlatformQueryQursorBase.cs | 8 +++
.../Impl/Client/Cache/CacheClient.cs | 2 +-
.../Cache/Query/ClientFieldsQueryCursor.cs | 12 +---
.../NuGet/LINQPad/BinaryModeExample.linq | 2 +-
.../NuGet/LINQPad/QueryExample.linq | 2 +-
.../Apache.Ignite.Linq/CacheLinqExtensions.cs | 10 +--
.../Impl/CacheFieldsQueryExecutor.cs | 10 +--
modules/platforms/dotnet/Apache.Ignite.ndproj | 2 +-
.../Datagrid/BinaryModeExample.cs | 5 +-
.../Datagrid/QueryDmlExample.cs | 18 ++---
.../Datagrid/QueryExample.cs | 5 +-
.../dotnet/examples/dotnetcore/SqlExample.cs | 2 +-
28 files changed, 224 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformAbstractQueryCursor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformAbstractQueryCursor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformAbstractQueryCursor.java
index 20e98fc..aef8e59 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformAbstractQueryCursor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformAbstractQueryCursor.java
@@ -176,6 +176,15 @@ public abstract class PlatformAbstractQueryCursor<T> extends PlatformAbstractTar
protected abstract void write(BinaryRawWriterEx writer, T val);
/**
+ * Gets the cursor.
+ *
+ * @return Cursor.
+ */
+ public QueryCursorEx<T> cursor() {
+ return cursor;
+ }
+
+ /**
* Query cursor consumer.
*/
private static class Consumer<T> implements QueryCursorEx.Consumer<T> {
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformFieldsQueryCursor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformFieldsQueryCursor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformFieldsQueryCursor.java
index 25f86f2..15561b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformFieldsQueryCursor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/query/PlatformFieldsQueryCursor.java
@@ -17,15 +17,21 @@
package org.apache.ignite.internal.processors.platform.cache.query;
-import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
import org.apache.ignite.internal.binary.BinaryRawWriterEx;
import org.apache.ignite.internal.processors.cache.query.QueryCursorEx;
import org.apache.ignite.internal.processors.platform.PlatformContext;
+import java.util.List;
+
/**
* Interop cursor for fields query.
*/
public class PlatformFieldsQueryCursor extends PlatformAbstractQueryCursor<List<?>> {
+ /** Gets field names. */
+ private static final int OP_GET_FIELD_NAMES = 7;
+
/**
* Constructor.
*
@@ -52,4 +58,20 @@ public class PlatformFieldsQueryCursor extends PlatformAbstractQueryCursor<List<
writer.writeInt(rowSizePos, rowEndPos - rowSizePos);
}
+
+ /** {@inheritDoc} */
+ @Override public void processOutStream(int type, final BinaryRawWriterEx writer) throws IgniteCheckedException {
+ if (type == OP_GET_FIELD_NAMES) {
+ FieldsQueryCursor fq = (FieldsQueryCursor) cursor();
+
+ int cnt = fq.getColumnsCount();
+ writer.writeInt(cnt);
+
+ for (int i = 0; i < cnt; i++) {
+ writer.writeString(fq.getFieldName(i));
+ }
+ } else {
+ super.processOutStream(type, writer);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
index 2fe3309..93e3a53 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/ExpiryCacheHolderTest.cs
@@ -419,6 +419,11 @@ namespace Apache.Ignite.AspNet.Tests
throw new NotImplementedException();
}
+ public IFieldsQueryCursor Query(SqlFieldsQuery qry)
+ {
+ throw new NotImplementedException();
+ }
+
public IQueryCursor<IList> QueryFields(SqlFieldsQuery qry)
{
throw new NotImplementedException();
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
index 9fa4538..34df6d7 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryFooterTest.cs
@@ -146,7 +146,7 @@ namespace Apache.Ignite.Core.Tests.Binary
Assert.AreEqual(dt.Arr, cache[1].Arr);
// SQL: read field on Java side to ensure correct offset handling.
- var res = cache.QueryFields(new SqlFieldsQuery("select int from OffsetTest")).GetAll()[0][0];
+ var res = cache.Query(new SqlFieldsQuery("select int from OffsetTest")).GetAll()[0][0];
Assert.AreEqual(dt.Int, (int) res);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/Serializable/SqlDmlTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/Serializable/SqlDmlTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/Serializable/SqlDmlTest.cs
index 4b77dbe..0ffd068 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/Serializable/SqlDmlTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/Serializable/SqlDmlTest.cs
@@ -107,7 +107,7 @@ namespace Apache.Ignite.Core.Tests.Binary.Serializable
// Test DML.
var guid = Guid.NewGuid();
- var insertRes = cache.QueryFields(new SqlFieldsQuery(
+ var insertRes = cache.Query(new SqlFieldsQuery(
"insert into SimpleSerializable(_key, Byte, Bool, Short, Int, Long, Float, Double, " +
"Decimal, Guid, String) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
3, 45, true, 43, 33, 99, 4.5f, 6.7, 9.04m, guid, "bar33")).GetAll();
@@ -141,7 +141,7 @@ namespace Apache.Ignite.Core.Tests.Binary.Serializable
Assert.AreEqual(uint.MaxValue, cache[1].Uint);
// Test SQL.
- var sqlRes = cache.QueryFields(new SqlFieldsQuery(
+ var sqlRes = cache.Query(new SqlFieldsQuery(
"select uint from DotNetSpecificSerializable where uint <> 0")).GetAll();
Assert.AreEqual(1, sqlRes.Count);
@@ -152,7 +152,7 @@ namespace Apache.Ignite.Core.Tests.Binary.Serializable
Assert.AreEqual(uint.MaxValue, linqRes);
// Test DML.
- var dmlRes = cache.QueryFields(new SqlFieldsQuery(
+ var dmlRes = cache.Query(new SqlFieldsQuery(
"insert into DotNetSpecificSerializable(_key, uint) values (?, ?), (?, ?)",
2, uint.MaxValue, 3, 88)).GetAll();
Assert.AreEqual(1, dmlRes.Count);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
index 5ba2fc9..0b1af41 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheTestAsyncWrapper.cs
@@ -445,6 +445,13 @@ namespace Apache.Ignite.Core.Tests.Cache
}
/** <inheritDoc /> */
+ public IFieldsQueryCursor Query(SqlFieldsQuery qry)
+ {
+ return _cache.Query(qry);
+ }
+
+ /** <inheritDoc /> */
+ [Obsolete]
public IQueryCursor<IList> QueryFields(SqlFieldsQuery qry)
{
return _cache.QueryFields(qry);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheDmlQueriesTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheDmlQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheDmlQueriesTest.cs
index 741dff0..f289e56 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheDmlQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheDmlQueriesTest.cs
@@ -76,7 +76,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cache = Ignition.GetIgnite().CreateCache<int, Foo>(cfg);
// Test insert.
- var res = cache.QueryFields(new SqlFieldsQuery("insert into foo(_key, id, name) " +
+ var res = cache.Query(new SqlFieldsQuery("insert into foo(_key, id, name) " +
"values (?, ?, ?), (?, ?, ?)",
1, 2, "John", 3, 4, "Mary")).GetAll();
@@ -118,7 +118,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cache = Ignition.GetIgnite().CreateCache<int, Foo>(cfg);
- var ex = Assert.Throws<IgniteException>(() => cache.QueryFields(new SqlFieldsQuery(
+ var ex = Assert.Throws<IgniteException>(() => cache.Query(new SqlFieldsQuery(
"insert into foo(_key, name) values (?, ?)", 1, "bar")).GetAll());
Assert.AreEqual("Null value is not allowed for column 'ID'", ex.Message);
@@ -133,7 +133,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cfg = new CacheConfiguration("not_null_attr", new QueryEntity(typeof(int), typeof(Foo)));
var cache = Ignition.GetIgnite().CreateCache<int, Foo>(cfg);
- var ex = Assert.Throws<IgniteException>(() => cache.QueryFields(new SqlFieldsQuery(
+ var ex = Assert.Throws<IgniteException>(() => cache.Query(new SqlFieldsQuery(
"insert into foo(_key, id) values (?, ?)", 1, 2)).GetAll());
Assert.AreEqual("Null value is not allowed for column 'NAME'", ex.Message);
@@ -169,7 +169,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
foreach (var val in vals)
{
- var res = cache.QueryFields(new SqlFieldsQuery(
+ var res = cache.Query(new SqlFieldsQuery(
"insert into string(_key, _val) values (?, ?)", val, val.ToString())).GetAll();
Assert.AreEqual(1, res.Count);
@@ -190,7 +190,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cache = Ignition.GetIgnite().CreateCache<Key, Foo>(cfg);
// Test insert.
- var res = cache.QueryFields(new SqlFieldsQuery("insert into foo(hi, lo, id, name) " +
+ var res = cache.Query(new SqlFieldsQuery("insert into foo(hi, lo, id, name) " +
"values (-1, 65500, 3, 'John'), (255, 128, 6, 'Mary')")).GetAll();
Assert.AreEqual(1, res.Count);
@@ -240,7 +240,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cache = Ignition.GetIgnite().CreateCache<Key2, Foo>(cfg);
// Test insert.
- var res = cache.QueryFields(new SqlFieldsQuery("insert into foo(hi, lo, str, id, name) " +
+ var res = cache.Query(new SqlFieldsQuery("insert into foo(hi, lo, str, id, name) " +
"values (1, 2, 'Фу', 3, 'John'), (4, 5, 'Бар', 6, 'Mary')")).GetAll();
Assert.AreEqual(1, res.Count);
@@ -293,7 +293,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cache = Ignition.GetIgnite().CreateCache<Key, Foo>(cfg);
var ex = Assert.Throws<IgniteException>(
- () => cache.QueryFields(new SqlFieldsQuery("insert into foo(lo, hi, id, name) " +
+ () => cache.Query(new SqlFieldsQuery("insert into foo(lo, hi, id, name) " +
"values (1, 2, 3, 'John'), (4, 5, 6, 'Mary')")));
Assert.AreEqual("Ownership flag not set for binary property. Have you set 'keyFields' " +
@@ -322,7 +322,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var cache = Ignition.GetIgnite().CreateCache<object, object>(cfg)
.WithKeepBinary<IBinaryObject, IBinaryObject>();
- var res = cache.QueryFields(new SqlFieldsQuery("insert into car(VIN, Id, Make, Year) " +
+ var res = cache.Query(new SqlFieldsQuery("insert into car(VIN, Id, Make, Year) " +
"values ('DLRDMC', 88, 'DeLorean', 1982)")).GetAll();
Assert.AreEqual(1, res.Count);
@@ -362,7 +362,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
};
// Test insert.
- var res = cache.QueryFields(new SqlFieldsQuery(
+ var res = cache.Query(new SqlFieldsQuery(
"insert into string(byte, sbyte, short, ushort, int, uint, long, ulong, float, double, decimal, " +
"guid, string, key, _val) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
key.Byte, key.SByte, key.Short, key.UShort, key.Int, key.UInt, key.Long, key.ULong, key.Float,
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
index 2e24ff6..9768fca 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesCodeConfigurationTest.cs
@@ -91,7 +91,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Assert.AreEqual(2, cursor.GetAll().Single().Key);
}
- using (var cursor = cache.QueryFields(new SqlFieldsQuery(
+ using (var cursor = cache.Query(new SqlFieldsQuery(
"select _key from CustomTableName where age > ? and birthday < ?", 10, DateTime.UtcNow)))
{
Assert.AreEqual(2, cursor.GetAll().Single()[0]);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
index 8134b3d..278d1fd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/CacheQueriesTest.cs
@@ -20,7 +20,6 @@
namespace Apache.Ignite.Core.Tests.Cache.Query
{
using System;
- using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
@@ -156,7 +155,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
public void TestValidationSqlFields()
{
// 1. No sql.
- Assert.Throws<ArgumentException>(() => { Cache().QueryFields(new SqlFieldsQuery(null)); });
+ Assert.Throws<ArgumentException>(() => { Cache().Query(new SqlFieldsQuery(null)); });
}
/// <summary>
@@ -285,11 +284,9 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Cache().Put(3, new QueryPerson("Sidorov", 50));
// 1. Empty result set.
- using (
- IQueryCursor<IList> cursor = Cache().QueryFields(
- new SqlFieldsQuery("SELECT age FROM QueryPerson WHERE age < ?", 50)))
+ using (var cursor = Cache().Query(new SqlFieldsQuery("SELECT age FROM QueryPerson WHERE age < ?", 50)))
{
- foreach (IList entry in cursor.GetAll())
+ foreach (var entry in cursor.GetAll())
Assert.IsTrue((int) entry[0] < 50);
}
}
@@ -390,7 +387,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Lazy = lazy
};
- using (IQueryCursor<IList> cursor = cache.QueryFields(qry))
+ using (var cursor = cache.Query(qry))
{
HashSet<int> exp0 = new HashSet<int>(exp);
@@ -403,9 +400,12 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
}
Assert.AreEqual(0, exp0.Count);
+ Assert.AreEqual(new[] {"NAME", "AGE"}, cursor.FieldNames);
}
- using (IQueryCursor<IList> cursor = cache.QueryFields(qry))
+ // Test old API as well.
+#pragma warning disable 618
+ using (var cursor = cache.QueryFields(qry))
{
HashSet<int> exp0 = new HashSet<int>(exp);
@@ -418,6 +418,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Assert.AreEqual(0, exp0.Count);
}
+#pragma warning restore 618
}
/// <summary>
@@ -622,7 +623,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
strings[1] = "foo";
// Default schema.
- var res = doubles.QueryFields(new SqlFieldsQuery(
+ var res = doubles.Query(new SqlFieldsQuery(
"select S._val from double as D join \"strings\".string as S on S._key = D._key"))
.Select(x => (string) x[0])
.Single();
@@ -630,7 +631,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Assert.AreEqual("foo", res);
// Custom schema.
- res = doubles.QueryFields(new SqlFieldsQuery(
+ res = doubles.Query(new SqlFieldsQuery(
"select S._val from \"doubles\".double as D join string as S on S._key = D._key")
{
Schema = strings.Name
@@ -667,13 +668,13 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
var sql = "select T0.Age from QueryPerson as T0 " +
"inner join QueryPerson as T1 on ((? - T1.Age - 1) = T0._key)";
- var res = cache.QueryFields(new SqlFieldsQuery(sql, count)).GetAll().Distinct().Count();
+ var res = cache.Query(new SqlFieldsQuery(sql, count)).GetAll().Distinct().Count();
Assert.Greater(res, 0);
Assert.Less(res, count);
// Test distributed join: returns complete results
- res = cache.QueryFields(new SqlFieldsQuery(sql, count) {EnableDistributedJoins = true})
+ res = cache.Query(new SqlFieldsQuery(sql, count) {EnableDistributedJoins = true})
.GetAll().Distinct().Count();
Assert.AreEqual(count, res);
@@ -709,7 +710,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
cache[1] = new QueryPerson("Joe", 48);
- var row = cache.QueryFields(new SqlFieldsQuery("select * from QueryPerson")).GetAll()[0];
+ var row = cache.Query(new SqlFieldsQuery("select * from QueryPerson")).GetAll()[0];
Assert.AreEqual(2, row.Count);
Assert.AreEqual(48, row[0]);
Assert.AreEqual("Joe", row[1]);
@@ -736,7 +737,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
cache[1] = new QueryPerson("John", 33);
- row = cache.QueryFields(new SqlFieldsQuery("select * from QueryPerson")).GetAll()[0];
+ row = cache.Query(new SqlFieldsQuery("select * from QueryPerson")).GetAll()[0];
Assert.AreEqual(3, row.Count);
Assert.AreEqual(33, row[0]);
@@ -746,7 +747,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
Assert.AreEqual("John", person.Name);
// Check explicit select.
- row = cache.QueryFields(new SqlFieldsQuery("select FullKey from QueryPerson")).GetAll()[0];
+ row = cache.Query(new SqlFieldsQuery("select FullKey from QueryPerson")).GetAll()[0];
Assert.AreEqual(1, row[0]);
}
@@ -786,11 +787,47 @@ namespace Apache.Ignite.Core.Tests.Cache.Query
};
// ReSharper disable once ReturnValueOfPureMethodIsNotUsed
- var ex = Assert.Throws<CacheException>(() => cache.QueryFields(fieldsQry).ToArray());
+ var ex = Assert.Throws<CacheException>(() => cache.Query(fieldsQry).ToArray());
Assert.IsTrue(ex.ToString().Contains("QueryCancelledException: The query was cancelled while executing."));
}
/// <summary>
+ /// Tests the FieldNames property.
+ /// </summary>
+ [Test]
+ public void TestFieldNames()
+ {
+ var cache = Cache();
+ PopulateCache(cache, false, 5, x => true);
+
+ // Get before iteration.
+ var qry = new SqlFieldsQuery("SELECT * FROM QueryPerson");
+ var cur = cache.Query(qry);
+ var names = cur.FieldNames;
+
+ Assert.AreEqual(new[] {"AGE", "NAME" }, names);
+
+ cur.Dispose();
+ Assert.AreSame(names, cur.FieldNames);
+
+ Assert.Throws<NotSupportedException>(() => cur.FieldNames.Add("x"));
+
+ // Custom order, key-val, get after iteration.
+ qry.Sql = "SELECT NAME, _key, AGE, _val FROM QueryPerson";
+ cur = cache.Query(qry);
+ cur.GetAll();
+
+ Assert.AreEqual(new[] { "NAME", "_KEY", "AGE", "_VAL" }, cur.FieldNames);
+
+ // Get after disposal.
+ qry.Sql = "SELECT 1, AGE FROM QueryPerson";
+ cur = cache.Query(qry);
+ cur.Dispose();
+
+ Assert.AreEqual(new[] { "1", "AGE" }, cur.FieldNames);
+ }
+
+ /// <summary>
/// Validates the query results.
/// </summary>
/// <param name="cache">Cache.</param>
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Introspection.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Introspection.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Introspection.cs
index 6e7483e..aa26548 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Introspection.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/Query/Linq/CacheLinqTest.Introspection.cs
@@ -72,7 +72,7 @@ namespace Apache.Ignite.Core.Tests.Cache.Query.Linq
Assert.AreEqual(new[] { 10 }, fq.Arguments);
Assert.IsTrue(fq.Local);
- Assert.AreEqual(PersonCount - 11, cache.QueryFields(fq).GetAll().Count);
+ Assert.AreEqual(PersonCount - 11, cache.Query(fq).GetAll().Count);
Assert.AreEqual(999, fq.PageSize);
Assert.IsFalse(fq.EnableDistributedJoins);
Assert.IsTrue(fq.EnforceJoinOrder);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
index 6f6df11..b9de090 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/SqlQueryTest.cs
@@ -114,7 +114,9 @@ namespace Apache.Ignite.Core.Tests.Client.Cache
// DateTime.
qry = new SqlFieldsQuery("select Id, DateTime from Person where DateTime > ?", DateTime.UtcNow.AddDays(9));
- Assert.AreEqual(cache[Count].DateTime, cache.Query(qry).Single().Last());
+ cursor = cache.Query(qry);
+ Assert.AreEqual(new[] {"ID", "DATETIME" }, cursor.FieldNames);
+ Assert.AreEqual(cache[Count].DateTime, cursor.Single().Last());
// Invalid args.
qry.Sql = null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
index f83f279..8a9fe59 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICache.cs
@@ -705,6 +705,14 @@ namespace Apache.Ignite.Core.Cache
/// </summary>
/// <param name="qry">SQL fields query.</param>
/// <returns>Cursor.</returns>
+ IFieldsQueryCursor Query(SqlFieldsQuery qry);
+
+ /// <summary>
+ /// Queries separate entry fields.
+ /// </summary>
+ /// <param name="qry">SQL fields query.</param>
+ /// <returns>Cursor.</returns>
+ [Obsolete("Use Query(SqlFieldsQuery qry) instead.")]
IQueryCursor<IList> QueryFields(SqlFieldsQuery qry);
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReaderExtensions.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReaderExtensions.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReaderExtensions.cs
index db2f84f..b13318c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReaderExtensions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Binary/BinaryReaderExtensions.cs
@@ -130,7 +130,7 @@ namespace Apache.Ignite.Core.Impl.Binary
/// <summary>
/// Reads the string collection.
/// </summary>
- public static ICollection<string> ReadStringCollection(this IBinaryRawReader reader)
+ public static List<string> ReadStringCollection(this IBinaryRawReader reader)
{
Debug.Assert(reader != null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
index ca9fb63..6661161 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheImpl.cs
@@ -1054,17 +1054,26 @@ namespace Apache.Ignite.Core.Impl.Cache
#region Queries
/** <inheritDoc /> */
+ public IFieldsQueryCursor Query(SqlFieldsQuery qry)
+ {
+ var cursor = QueryFieldsInternal(qry);
+
+ return new FieldsQueryCursor(cursor, _flagKeepBinary,
+ (reader, count) => ReadFieldsArrayList(reader, count));
+ }
+
+ /** <inheritDoc /> */
public IQueryCursor<IList> QueryFields(SqlFieldsQuery qry)
{
- return QueryFields(qry, ReadFieldsArrayList);
+ return Query(qry, (reader, count) => (IList) ReadFieldsArrayList(reader, count));
}
/// <summary>
/// Reads the fields array list.
/// </summary>
- private static IList ReadFieldsArrayList(IBinaryRawReader reader, int count)
+ private static List<object> ReadFieldsArrayList(IBinaryRawReader reader, int count)
{
- IList res = new ArrayList(count);
+ var res = new List<object>(count);
for (var i = 0; i < count; i++)
res.Add(reader.ReadObject<object>());
@@ -1073,15 +1082,21 @@ namespace Apache.Ignite.Core.Impl.Cache
}
/** <inheritDoc /> */
- public IQueryCursor<T> QueryFields<T>(SqlFieldsQuery qry, Func<IBinaryRawReader, int, T> readerFunc)
+ public IQueryCursor<T> Query<T>(SqlFieldsQuery qry, Func<IBinaryRawReader, int, T> readerFunc)
+ {
+ var cursor = QueryFieldsInternal(qry);
+
+ return new FieldsQueryCursor<T>(cursor, _flagKeepBinary, readerFunc);
+ }
+
+ private IPlatformTargetInternal QueryFieldsInternal(SqlFieldsQuery qry)
{
IgniteArgumentCheck.NotNull(qry, "qry");
- IgniteArgumentCheck.NotNull(readerFunc, "readerFunc");
if (string.IsNullOrEmpty(qry.Sql))
throw new ArgumentException("Sql cannot be null or empty");
- var cursor = DoOutOpObject((int) CacheOp.QrySqlFields, writer =>
+ return DoOutOpObject((int) CacheOp.QrySqlFields, writer =>
{
writer.WriteBoolean(qry.Local);
writer.WriteString(qry.Sql);
@@ -1097,8 +1112,6 @@ namespace Apache.Ignite.Core.Impl.Cache
writer.WriteBoolean(qry.Colocated);
writer.WriteString(qry.Schema); // Schema
});
-
- return new FieldsQueryCursor<T>(cursor, _flagKeepBinary, readerFunc);
}
/** <inheritDoc /> */
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/ICacheInternal.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/ICacheInternal.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/ICacheInternal.cs
index 1ec5341..fb968cf 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/ICacheInternal.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/ICacheInternal.cs
@@ -36,7 +36,7 @@ namespace Apache.Ignite.Core.Impl.Cache
/// <returns>
/// Cursor.
/// </returns>
- IQueryCursor<T> QueryFields<T>(SqlFieldsQuery qry, Func<IBinaryRawReader, int, T> readerFunc);
+ IQueryCursor<T> Query<T>(SqlFieldsQuery qry, Func<IBinaryRawReader, int, T> readerFunc);
/// <summary>
/// Invokes a cache extension.
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs
index c60e010..c9e64a8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/FieldsQueryCursor.cs
@@ -18,7 +18,11 @@
namespace Apache.Ignite.Core.Impl.Cache.Query
{
using System;
+ using System.Collections.Generic;
+ using System.Collections.ObjectModel;
using Apache.Ignite.Core.Binary;
+ using Apache.Ignite.Core.Cache.Query;
+ using Apache.Ignite.Core.Impl.Binary;
/// <summary>
/// Cursor for entry-based queries.
@@ -47,4 +51,39 @@ namespace Apache.Ignite.Core.Impl.Cache.Query
// No-op.
}
}
+
+ /// <summary>
+ /// Cursor for entry-based queries.
+ /// </summary>
+ internal class FieldsQueryCursor : FieldsQueryCursor<IList<object>>, IFieldsQueryCursor
+ {
+ /** */
+ private const int OpGetFieldNames = 7;
+
+ /** */
+ private IList<string> _fieldNames;
+
+ /// <summary>
+ /// Constructor.
+ /// </summary>
+ /// <param name="target">Target.</param>
+ /// <param name="keepBinary">Keep poratble flag.</param>
+ /// <param name="readerFunc">The reader function.</param>
+ public FieldsQueryCursor(IPlatformTargetInternal target, bool keepBinary,
+ Func<IBinaryRawReader, int, IList<object>> readerFunc) : base(target, keepBinary, readerFunc)
+ {
+ // No-op.
+ }
+
+ /** <inheritdoc /> */
+ public IList<string> FieldNames
+ {
+ get
+ {
+ return _fieldNames ??
+ (_fieldNames = new ReadOnlyCollection<string>(
+ Target.OutStream(OpGetFieldNames, reader => reader.ReadStringCollection())));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/PlatformQueryQursorBase.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/PlatformQueryQursorBase.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/PlatformQueryQursorBase.cs
index fc78392..3f1e4fc 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/PlatformQueryQursorBase.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/Query/PlatformQueryQursorBase.cs
@@ -54,6 +54,14 @@ namespace Apache.Ignite.Core.Impl.Cache.Query
_target = target;
}
+ /// <summary>
+ /// Gets the target.
+ /// </summary>
+ public IPlatformTargetInternal Target
+ {
+ get { return _target; }
+ }
+
/** <inheritdoc /> */
protected override IList<T> GetAllInternal()
{
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/CacheClient.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/CacheClient.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/CacheClient.cs
index 93829c2..2344417 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/CacheClient.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/CacheClient.cs
@@ -196,7 +196,7 @@ namespace Apache.Ignite.Core.Impl.Client.Cache
}
/** <inheritDoc /> */
- public IQueryCursor<T> QueryFields<T>(SqlFieldsQuery sqlFieldsQuery, Func<IBinaryRawReader, int, T> readerFunc)
+ public IQueryCursor<T> Query<T>(SqlFieldsQuery sqlFieldsQuery, Func<IBinaryRawReader, int, T> readerFunc)
{
return DoOutInOp(ClientOp.QuerySqlFields,
w => WriteSqlFieldsQuery(w, sqlFieldsQuery, false),
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/Query/ClientFieldsQueryCursor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/Query/ClientFieldsQueryCursor.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/Query/ClientFieldsQueryCursor.cs
index 2e57863..c8f7e1f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/Query/ClientFieldsQueryCursor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/Cache/Query/ClientFieldsQueryCursor.cs
@@ -22,6 +22,7 @@ namespace Apache.Ignite.Core.Impl.Client.Cache.Query
using System.Diagnostics;
using Apache.Ignite.Core.Binary;
using Apache.Ignite.Core.Cache.Query;
+ using Apache.Ignite.Core.Impl.Binary;
using Apache.Ignite.Core.Impl.Binary.IO;
/// <summary>
@@ -64,16 +65,9 @@ namespace Apache.Ignite.Core.Impl.Client.Cache.Query
/// <summary>
/// Reads the columns.
/// </summary>
- internal static string[] ReadColumns(IBinaryRawReader reader)
+ internal static List<string> ReadColumns(IBinaryRawReader reader)
{
- var res = new string[reader.ReadInt()];
-
- for (var i = 0; i < res.Length; i++)
- {
- res[i] = reader.ReadString();
- }
-
- return res;
+ return reader.ReadStringCollection();
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/BinaryModeExample.linq
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/BinaryModeExample.linq b/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/BinaryModeExample.linq
index 2220650..4b2291e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/BinaryModeExample.linq
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/BinaryModeExample.linq
@@ -90,6 +90,6 @@ using (var ignite = Ignition.Start())
.Dump("Persons with age less than 40:");
// Run SQL fields query.
- cache.QueryFields(new SqlFieldsQuery("select name from Person order by name"))
+ cache.Query(new SqlFieldsQuery("select name from Person order by name"))
.Dump("All person names:");
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/QueryExample.linq
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/QueryExample.linq b/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/QueryExample.linq
index 19d55a3..b3d63ba 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/QueryExample.linq
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/NuGet/LINQPad/QueryExample.linq
@@ -66,7 +66,7 @@ void Main()
.Dump("Persons working for " + orgName);
// Fields query
- orgs.QueryFields(new SqlFieldsQuery("select name, size from Organization")).Dump("Fields query");
+ orgs.Query(new SqlFieldsQuery("select name, size from Organization")).Dump("Fields query");
// Full text query
persons.Query(new TextQuery(typeof(Person), "Chris*")).Dump("Persons starting with 'Chris'");
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
index 940b23b..935666f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/CacheLinqExtensions.cs
@@ -23,6 +23,7 @@ namespace Apache.Ignite.Linq
using System.Linq.Expressions;
using Apache.Ignite.Core.Cache;
using Apache.Ignite.Core.Cache.Configuration;
+ using Apache.Ignite.Core.Cache.Query;
using Apache.Ignite.Core.Impl.Cache;
using Apache.Ignite.Core.Impl.Common;
using Apache.Ignite.Linq.Impl;
@@ -37,8 +38,7 @@ namespace Apache.Ignite.Linq
/// Gets an <see cref="IQueryable{T}"/> instance over this cache.
/// <para />
/// Resulting query will be translated to cache SQL query and executed over the cache instance
- /// via either <see cref="ICache{TK,TV}.Query"/> or <see cref="ICache{TK,TV}.QueryFields"/>,
- /// depending on requested result.
+ /// via <see cref="ICache{TK,TV}.Query(SqlFieldsQuery)"/>.
/// <para />
/// Result of this method (and subsequent query) can be cast to <see cref="ICacheQueryable"/>
/// for introspection, or converted with <see cref="ToCacheQueryable{T}"/> extension method.
@@ -59,7 +59,7 @@ namespace Apache.Ignite.Linq
/// Gets an <see cref="IQueryable{T}"/> instance over this cache.
/// <para />
/// Resulting query will be translated to cache SQL query and executed over the cache instance
- /// via either <see cref="ICache{TK,TV}.Query"/> or <see cref="ICache{TK,TV}.QueryFields"/>,
+ /// via <see cref="ICache{TK,TV}.Query(SqlFieldsQuery)"/>.
/// depending on requested result.
/// <para />
/// Result of this method (and subsequent query) can be cast to <see cref="ICacheQueryable"/> for introspection.
@@ -82,7 +82,7 @@ namespace Apache.Ignite.Linq
/// Gets an <see cref="IQueryable{T}" /> instance over this cache.
/// <para />
/// Resulting query will be translated to cache SQL query and executed over the cache instance
- /// via either <see cref="ICache{TK,TV}.Query" /> or <see cref="ICache{TK,TV}.QueryFields" />,
+ /// via <see cref="ICache{TK,TV}.Query(SqlFieldsQuery)"/>.
/// depending on requested result.
/// <para />
/// Result of this method (and subsequent query) can be cast to <see cref="ICacheQueryable" /> for introspection.
@@ -112,7 +112,7 @@ namespace Apache.Ignite.Linq
/// Gets an <see cref="IQueryable{T}" /> instance over this cache.
/// <para />
/// Resulting query will be translated to cache SQL query and executed over the cache instance
- /// via either <see cref="ICache{TK,TV}.Query" /> or <see cref="ICache{TK,TV}.QueryFields" />,
+ /// via <see cref="ICache{TK,TV}.Query(SqlFieldsQuery)"/>.
/// depending on requested result.
/// <para />
/// Result of this method (and subsequent query) can be cast to <see cref="ICacheQueryable" /> for introspection.
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryExecutor.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryExecutor.cs b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryExecutor.cs
index fb34270..9d0edd7 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryExecutor.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Linq/Impl/CacheFieldsQueryExecutor.cs
@@ -89,7 +89,7 @@ namespace Apache.Ignite.Linq.Impl
var selector = GetResultSelector<T>(queryModel.SelectClause.Selector);
- return _cache.QueryFields(qry, selector);
+ return _cache.Query(qry, selector);
}
/// <summary>
@@ -102,7 +102,7 @@ namespace Apache.Ignite.Linq.Impl
var qryText = GetQueryData(queryModel).QueryText;
var selector = GetResultSelector<T>(queryModel.SelectClause.Selector);
- return args => _cache.QueryFields(GetFieldsQuery(qryText, args), selector);
+ return args => _cache.Query(GetFieldsQuery(qryText, args), selector);
}
/// <summary>
@@ -144,7 +144,7 @@ namespace Apache.Ignite.Linq.Impl
// Simple case: lambda with no parameters. Only embedded parameters are used.
if (!queryLambda.Parameters.Any())
{
- return argsUnused => _cache.QueryFields(GetFieldsQuery(qryText, qryParams), selector);
+ return argsUnused => _cache.Query(GetFieldsQuery(qryText, qryParams), selector);
}
// These are in order of usage in query
@@ -157,7 +157,7 @@ namespace Apache.Ignite.Linq.Impl
if (qryOrderArgs.Length == qryParams.Length
&& qryOrderArgs.SequenceEqual(userOrderArgs))
{
- return args => _cache.QueryFields(GetFieldsQuery(qryText, args), selector);
+ return args => _cache.Query(GetFieldsQuery(qryText, args), selector);
}
// General case: embedded args and lambda args are mixed; same args can be used multiple times.
@@ -172,7 +172,7 @@ namespace Apache.Ignite.Linq.Impl
return -1;
}).ToArray();
- return args => _cache.QueryFields(
+ return args => _cache.Query(
GetFieldsQuery(qryText, MapQueryArgs(args, qryParams, mapping)), selector);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/Apache.Ignite.ndproj
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.ndproj b/modules/platforms/dotnet/Apache.Ignite.ndproj
index 5ed9e3c..ae99617 100644
--- a/modules/platforms/dotnet/Apache.Ignite.ndproj
+++ b/modules/platforms/dotnet/Apache.Ignite.ndproj
@@ -2029,7 +2029,7 @@ select new {
// is equal to 3 minutes per derived class used by the base class +
// 3 minutes per member of a derived class used by the base class.
//</HowToFix>]]></Query>
- <Query Active="True" DisplayList="True" DisplayStat="False" DisplaySelectionView="False" IsCriticalRule="False"><![CDATA[// <Name>Class shouldn't be too deep in inheritance tree</Name>
+ <Query Active="False" DisplayList="True" DisplayStat="False" DisplaySelectionView="False" IsCriticalRule="False"><![CDATA[// <Name>Class shouldn't be too deep in inheritance tree</Name>
warnif count > 0 from t in JustMyCode.Types
where t.IsClass
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/BinaryModeExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/BinaryModeExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/BinaryModeExample.cs
index 1cc406b..89021e6 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/BinaryModeExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/BinaryModeExample.cs
@@ -18,7 +18,6 @@
namespace Apache.Ignite.Examples.Datagrid
{
using System;
- using System.Collections;
using Apache.Ignite.Core;
using Apache.Ignite.Core.Binary;
using Apache.Ignite.Core.Cache;
@@ -196,12 +195,12 @@ namespace Apache.Ignite.Examples.Datagrid
/// <param name="cache">Cache.</param>
private static void SqlFieldsQueryExample(ICache<int, IBinaryObject> cache)
{
- var qry = cache.QueryFields(new SqlFieldsQuery("select name from Person order by name"));
+ var qry = cache.Query(new SqlFieldsQuery("select name from Person order by name"));
Console.WriteLine();
Console.WriteLine(">>> All person names:");
- foreach (IList row in qry)
+ foreach (var row in qry)
Console.WriteLine(">>> " + row[0]);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryDmlExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryDmlExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryDmlExample.cs
index 10bf1dd..00486bd 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryDmlExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryDmlExample.cs
@@ -98,7 +98,7 @@ namespace Apache.Ignite.Examples.Datagrid
EnableDistributedJoins = true
};
- using (var cursor = employeeCache.QueryFields(qry))
+ using (var cursor = employeeCache.Query(qry))
{
foreach (var row in cursor)
{
@@ -116,25 +116,25 @@ namespace Apache.Ignite.Examples.Datagrid
{
// Insert organizations.
var qry = new SqlFieldsQuery("insert into Organization (_key, name) values (?, ?)", 1, "ASF");
- organizationCache.QueryFields(qry);
+ organizationCache.Query(qry);
qry.Arguments = new object[] {2, "Eclipse"};
- organizationCache.QueryFields(qry);
+ organizationCache.Query(qry);
// Insert employees.
qry = new SqlFieldsQuery("insert into Employee (_key, name, organizationId, salary) values (?, ?, ?, ?)");
qry.Arguments = new object[] {1, "John Doe", 1, 4000};
- employeeCache.QueryFields(qry);
+ employeeCache.Query(qry);
qry.Arguments = new object[] {2, "Jane Roe", 1, 5000};
- employeeCache.QueryFields(qry);
+ employeeCache.Query(qry);
qry.Arguments = new object[] {3, "Mary Major", 2, 2000};
- employeeCache.QueryFields(qry);
+ employeeCache.Query(qry);
qry.Arguments = new object[] {4, "Richard Miles", 2, 3000};
- employeeCache.QueryFields(qry);
+ employeeCache.Query(qry);
}
/// <summary>
@@ -145,7 +145,7 @@ namespace Apache.Ignite.Examples.Datagrid
{
var qry = new SqlFieldsQuery("update Employee set salary = salary * 1.1 where organizationId = ?", 1);
- employeeCache.QueryFields(qry);
+ employeeCache.Query(qry);
}
/// <summary>
@@ -159,7 +159,7 @@ namespace Apache.Ignite.Examples.Datagrid
"select emp._key from Employee emp, \"{0}\".Organization org " +
"where org.Name != ? and org._key = emp.organizationId)", OrganizationCacheName), "ASF");
- employeeCache.QueryFields(qry);
+ employeeCache.Query(qry);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
index 98f9f50..23736ea 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/QueryExample.cs
@@ -18,7 +18,6 @@
namespace Apache.Ignite.Examples.Datagrid
{
using System;
- using System.Collections;
using Apache.Ignite.Core;
using Apache.Ignite.Core.Cache;
using Apache.Ignite.Core.Cache.Affinity;
@@ -182,12 +181,12 @@ namespace Apache.Ignite.Examples.Datagrid
/// <param name="cache">Cache.</param>
private static void SqlFieldsQueryExample(ICache<int, Employee> cache)
{
- var qry = cache.QueryFields(new SqlFieldsQuery("select name, salary from Employee"));
+ var qry = cache.Query(new SqlFieldsQuery("select name, salary from Employee"));
Console.WriteLine();
Console.WriteLine(">>> Employee names and their salaries:");
- foreach (IList row in qry)
+ foreach (var row in qry)
Console.WriteLine(">>> [Name=" + row[0] + ", salary=" + row[1] + ']');
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/8f327a18/modules/platforms/dotnet/examples/dotnetcore/SqlExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/dotnetcore/SqlExample.cs b/modules/platforms/dotnet/examples/dotnetcore/SqlExample.cs
index 1fb2e90..1fe6a74 100644
--- a/modules/platforms/dotnet/examples/dotnetcore/SqlExample.cs
+++ b/modules/platforms/dotnet/examples/dotnetcore/SqlExample.cs
@@ -139,7 +139,7 @@ namespace Apache.Ignite.Examples
/// <param name="cache">Cache.</param>
private static void SqlFieldsQueryExample(ICache<int, Employee> cache)
{
- var qry = cache.QueryFields(new SqlFieldsQuery("select name, salary from Employee"));
+ var qry = cache.Query(new SqlFieldsQuery("select name, salary from Employee"));
Console.WriteLine();
Console.WriteLine(">>> Employee names and their salaries:");
[23/50] [abbrv] ignite git commit: IGNITE-7079: Add examples for kNN
classification and for kNN regression
Posted by sb...@apache.org.
IGNITE-7079: Add examples for kNN classification and for kNN regression
this closes #3218
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/da782958
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/da782958
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/da782958
Branch: refs/heads/ignite-zk
Commit: da782958adad85b750ffdc39266644c5750d0f2f
Parents: 03bb551
Author: zaleslaw <za...@gmail.com>
Authored: Thu Dec 14 20:27:15 2017 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Thu Dec 14 20:27:15 2017 +0300
----------------------------------------------------------------------
.../KNNClassificationExample.java | 151 ++++++++++++++
.../ml/knn/classification/package-info.java | 22 ++
.../ignite/examples/ml/knn/package-info.java | 22 ++
.../ml/knn/regression/KNNRegressionExample.java | 152 ++++++++++++++
.../ml/knn/regression/package-info.java | 22 ++
.../src/main/resources/datasets/knn/README.md | 2 +
.../resources/datasets/knn/cleared_machines.txt | 209 +++++++++++++++++++
.../src/main/resources/datasets/knn/iris.txt | 150 +++++++++++++
.../ignite/ml/structures/LabeledDataset.java | 18 ++
.../structures/LabeledDatasetTestTrainPair.java | 116 ++++++++++
.../ignite/ml/knn/LabeledDatasetTest.java | 58 +++++
11 files changed, 922 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
new file mode 100644
index 0000000..a92e9af
--- /dev/null
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/KNNClassificationExample.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.knn.classification;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ExampleNodeStartup;
+import org.apache.ignite.ml.knn.models.KNNModel;
+import org.apache.ignite.ml.knn.models.KNNStrategy;
+import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * <p>
+ * Example of using {@link KNNModel} with iris dataset.</p>
+ * <p>
+ * Note that in this example we cannot guarantee order in which nodes return results of intermediate
+ * computations and therefore algorithm can return different results.</p>
+ * <p>
+ * Remote nodes should always be started with special configuration file which
+ * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
+ * <p>
+ * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
+ * with {@code examples/config/example-ignite.xml} configuration.</p>
+ */
+public class KNNClassificationExample {
+ /** Separator. */
+ private static final String SEPARATOR = "\t";
+
+ /** Path to the Iris dataset. */
+ static final String KNN_IRIS_TXT = "datasets/knn/iris.txt";
+
+ /**
+ * Executes example.
+ *
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String[] args) throws InterruptedException {
+ System.out.println(">>> kNN classification example started.");
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ KNNClassificationExample.class.getSimpleName(), () -> {
+
+ try {
+ // Prepare path to read
+ Path path = Paths.get(KNNClassificationExample.class.getClassLoader().getResource(KNN_IRIS_TXT).toURI());
+
+ // Read dataset from file
+ LabeledDataset dataset = LabeledDataset.loadTxt(path, SEPARATOR, true, false);
+
+ // Random splitting of iris data as 70% train and 30% test datasets
+ LabeledDatasetTestTrainPair split = new LabeledDatasetTestTrainPair(dataset, 0.3);
+
+ System.out.println("\n>>> Amount of observations in train dataset " + split.train().rowSize());
+ System.out.println("\n>>> Amount of observations in test dataset " + split.test().rowSize());
+
+ LabeledDataset test = split.test();
+ LabeledDataset train = split.train();
+
+ KNNModel knnMdl = new KNNModel(5, new EuclideanDistance(), KNNStrategy.SIMPLE, train);
+
+ // Clone labels
+ final double[] labels = test.labels();
+
+ // Save predicted classes to test dataset
+ for (int i = 0; i < test.rowSize(); i++) {
+ double predictedCls = knnMdl.predict(test.getRow(i).features());
+ test.setLabel(i, predictedCls);
+ }
+
+ // Calculate amount of errors on test dataset
+ int amountOfErrors = 0;
+ for (int i = 0; i < test.rowSize(); i++) {
+ if (test.label(i) != labels[i])
+ amountOfErrors++;
+ }
+
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + amountOfErrors / (double)test.rowSize());
+
+ // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+ int[][] confusionMtx = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
+ for (int i = 0; i < test.rowSize(); i++) {
+ int idx1 = (int)test.label(i);
+ int idx2 = (int)labels[i];
+ confusionMtx[idx1 - 1][idx2 - 1]++;
+ }
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+
+ // Calculate precision, recall and F-metric for each class
+ for (int i = 0; i < 3; i++) {
+ double precision = 0.0;
+ for (int j = 0; j < 3; j++)
+ precision += confusionMtx[i][j];
+ precision = confusionMtx[i][i] / precision;
+
+ double clsLb = (double)(i + 1);
+
+ System.out.println("\n>>> Precision for class " + clsLb + " is " + precision);
+
+ double recall = 0.0;
+ for (int j = 0; j < 3; j++)
+ recall += confusionMtx[j][i];
+ recall = confusionMtx[i][i] / recall;
+ System.out.println("\n>>> Recall for class " + clsLb + " is " + recall);
+
+ double fScore = 2 * precision * recall / (precision + recall);
+ System.out.println("\n>>> F-score for class " + clsLb + " is " + fScore);
+ }
+
+ }
+ catch (URISyntaxException | IOException e) {
+ e.printStackTrace();
+ System.out.println("\n>>> Check resources");
+ }
+ finally {
+ System.out.println("\n>>> kNN classification example completed.");
+ }
+
+ });
+
+ igniteThread.start();
+ igniteThread.join();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/package-info.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/package-info.java
new file mode 100644
index 0000000..d853f0d
--- /dev/null
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/classification/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * kNN classification examples.
+ */
+package org.apache.ignite.examples.ml.knn.classification;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/ml/org/apache/ignite/examples/ml/knn/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/package-info.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/package-info.java
new file mode 100644
index 0000000..8de4656
--- /dev/null
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * kNN examples.
+ */
+package org.apache.ignite.examples.ml.knn;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
new file mode 100644
index 0000000..f4a9e1c
--- /dev/null
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/KNNRegressionExample.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.knn.regression;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.examples.ExampleNodeStartup;
+import org.apache.ignite.examples.ml.knn.classification.KNNClassificationExample;
+import org.apache.ignite.ml.knn.models.KNNStrategy;
+import org.apache.ignite.ml.knn.models.Normalization;
+import org.apache.ignite.ml.knn.regression.KNNMultipleLinearRegression;
+import org.apache.ignite.ml.math.distances.ManhattanDistance;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * <p>
+ * Example of using {@link KNNMultipleLinearRegression} with iris dataset.</p>
+ * <p>
+ * Note that in this example we cannot guarantee order in which nodes return results of intermediate
+ * computations and therefore algorithm can return different results.</p>
+ * <p>
+ * Remote nodes should always be started with special configuration file which
+ * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
+ * <p>
+ * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
+ * with {@code examples/config/example-ignite.xml} configuration.</p>
+ */
+public class KNNRegressionExample {
+ /** Separator. */
+ private static final String SEPARATOR = ",";
+
+ /** */
+ public static final String KNN_CLEARED_MACHINES_TXT = "datasets/knn/cleared_machines.txt";
+
+ /**
+ * Executes example.
+ *
+ * @param args Command line arguments, none required.
+ */
+ public static void main(String[] args) throws InterruptedException {
+ System.out.println(">>> kNN regression example started.");
+ // Start ignite grid.
+ try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+ System.out.println(">>> Ignite grid started.");
+
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ KNNRegressionExample.class.getSimpleName(), () -> {
+
+ try {
+ // Prepare path to read
+ Path path = Paths.get(KNNClassificationExample.class.getClassLoader().getResource(KNN_CLEARED_MACHINES_TXT).toURI());
+
+ // Read dataset from file
+ LabeledDataset dataset = LabeledDataset.loadTxt(path, SEPARATOR, false, false);
+
+ // Normalize dataset
+ dataset.normalizeWith(Normalization.MINIMAX);
+
+ // Random splitting of iris data as 80% train and 20% test datasets
+ LabeledDatasetTestTrainPair split = new LabeledDatasetTestTrainPair(dataset, 0.2);
+
+ System.out.println("\n>>> Amount of observations in train dataset " + split.train().rowSize());
+ System.out.println("\n>>> Amount of observations in test dataset " + split.test().rowSize());
+
+ LabeledDataset test = split.test();
+ LabeledDataset train = split.train();
+
+ // Builds weighted kNN-regression with Manhattan Distance
+ KNNMultipleLinearRegression knnMdl = new KNNMultipleLinearRegression(7, new ManhattanDistance(), KNNStrategy.WEIGHTED, train);
+
+ // Clone labels
+ final double[] labels = test.labels();
+
+ // Save predicted classes to test dataset
+ for (int i = 0; i < test.rowSize(); i++) {
+ double predictedCls = knnMdl.predict(test.getRow(i).features());
+ test.setLabel(i, predictedCls);
+ }
+
+ // Calculate mean squared error (MSE)
+ double mse = 0.0;
+
+ for (int i = 0; i < test.rowSize(); i++)
+ mse += Math.pow(test.label(i) - labels[i], 2.0);
+ mse = mse / test.rowSize();
+
+ System.out.println("\n>>> Mean squared error (MSE) " + mse);
+
+ // Calculate mean absolute error (MAE)
+ double mae = 0.0;
+
+ for (int i = 0; i < test.rowSize(); i++)
+ mae += Math.abs(test.label(i) - labels[i]);
+ mae = mae / test.rowSize();
+
+ System.out.println("\n>>> Mean absolute error (MAE) " + mae);
+
+ // Calculate R^2 as 1 - RSS/TSS
+ double avg = 0.0;
+
+ for (int i = 0; i < test.rowSize(); i++)
+ avg += test.label(i);
+
+ avg = avg / test.rowSize();
+
+ double detCf = 0.0;
+ double tss = 0.0;
+
+ for (int i = 0; i < test.rowSize(); i++) {
+ detCf += Math.pow(test.label(i) - labels[i], 2.0);
+ tss += Math.pow(test.label(i) - avg, 2.0);
+ }
+
+ detCf = 1 - detCf / tss;
+
+ System.out.println("\n>>> R^2 " + detCf);
+ }
+ catch (URISyntaxException | IOException e) {
+ e.printStackTrace();
+ System.out.println("\n>>> Check resources");
+ }
+ finally {
+ System.out.println("\n>>> kNN regression example completed.");
+ }
+ });
+
+ igniteThread.start();
+ igniteThread.join();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/package-info.java
----------------------------------------------------------------------
diff --git a/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/package-info.java b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/package-info.java
new file mode 100644
index 0000000..e7ac336
--- /dev/null
+++ b/examples/src/main/ml/org/apache/ignite/examples/ml/knn/regression/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * kNN regression examples.
+ */
+package org.apache.ignite.examples.ml.knn.regression;
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/resources/datasets/knn/README.md
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/datasets/knn/README.md b/examples/src/main/resources/datasets/knn/README.md
new file mode 100644
index 0000000..2f9c5ec
--- /dev/null
+++ b/examples/src/main/resources/datasets/knn/README.md
@@ -0,0 +1,2 @@
+iris.txt and cleared_machines are from Lichman, M. (2013). UCI Machine Learning Repository [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, School of Information and Computer Science.
+Read more about machine dataset http://archive.ics.uci.edu/ml/machine-learning-databases/cpu-performance/machine.names
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/resources/datasets/knn/cleared_machines.txt
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/datasets/knn/cleared_machines.txt b/examples/src/main/resources/datasets/knn/cleared_machines.txt
new file mode 100644
index 0000000..cf8b6b0
--- /dev/null
+++ b/examples/src/main/resources/datasets/knn/cleared_machines.txt
@@ -0,0 +1,209 @@
+199,125,256,6000,256,16,128
+253,29,8000,32000,32,8,32
+253,29,8000,32000,32,8,32
+253,29,8000,32000,32,8,32
+132,29,8000,16000,32,8,16
+290,26,8000,32000,64,8,32
+381,23,16000,32000,64,16,32
+381,23,16000,32000,64,16,32
+749,23,16000,64000,64,16,32
+1238,23,32000,64000,128,32,64
+23,400,1000,3000,0,1,2
+24,400,512,3500,4,1,6
+70,60,2000,8000,65,1,8
+117,50,4000,16000,65,1,8
+15,350,64,64,0,1,4
+64,200,512,16000,0,4,32
+23,167,524,2000,8,4,15
+29,143,512,5000,0,7,32
+22,143,1000,2000,0,5,16
+124,110,5000,5000,142,8,64
+35,143,1500,6300,0,5,32
+39,143,3100,6200,0,5,20
+40,143,2300,6200,0,6,64
+45,110,3100,6200,0,6,64
+28,320,128,6000,0,1,12
+21,320,512,2000,4,1,3
+28,320,256,6000,0,1,6
+22,320,256,3000,4,1,3
+28,320,512,5000,4,1,5
+27,320,256,5000,4,1,6
+102,25,1310,2620,131,12,24
+102,25,1310,2620,131,12,24
+74,50,2620,10480,30,12,24
+74,50,2620,10480,30,12,24
+138,56,5240,20970,30,12,24
+136,64,5240,20970,30,12,24
+23,50,500,2000,8,1,4
+29,50,1000,4000,8,1,5
+44,50,2000,8000,8,1,5
+30,50,1000,4000,8,3,5
+41,50,1000,8000,8,3,5
+74,50,2000,16000,8,3,5
+74,50,2000,16000,8,3,6
+74,50,2000,16000,8,3,6
+54,133,1000,12000,9,3,12
+41,133,1000,8000,9,3,12
+18,810,512,512,8,1,1
+28,810,1000,5000,0,1,1
+36,320,512,8000,4,1,5
+38,200,512,8000,8,1,8
+34,700,384,8000,0,1,1
+19,700,256,2000,0,1,1
+72,140,1000,16000,16,1,3
+36,200,1000,8000,0,1,2
+30,110,1000,4000,16,1,2
+56,110,1000,12000,16,1,2
+42,220,1000,8000,16,1,2
+34,800,256,8000,0,1,4
+34,800,256,8000,0,1,4
+34,800,256,8000,0,1,4
+34,800,256,8000,0,1,4
+34,800,256,8000,0,1,4
+19,125,512,1000,0,8,20
+75,75,2000,8000,64,1,38
+113,75,2000,16000,64,1,38
+157,75,2000,16000,128,1,38
+18,90,256,1000,0,3,10
+20,105,256,2000,0,3,10
+28,105,1000,4000,0,3,24
+33,105,2000,4000,8,3,19
+47,75,2000,8000,8,3,24
+54,75,3000,8000,8,3,48
+20,175,256,2000,0,3,24
+23,300,768,3000,0,6,24
+25,300,768,3000,6,6,24
+52,300,768,12000,6,6,24
+27,300,768,4500,0,1,24
+50,300,384,12000,6,1,24
+18,300,192,768,6,6,24
+53,180,768,12000,6,1,31
+23,330,1000,3000,0,2,4
+30,300,1000,4000,8,3,64
+73,300,1000,16000,8,2,112
+20,330,1000,2000,0,1,2
+25,330,1000,4000,0,3,6
+28,140,2000,4000,0,3,6
+29,140,2000,4000,0,4,8
+32,140,2000,4000,8,1,20
+175,140,2000,32000,32,1,20
+57,140,2000,8000,32,1,54
+181,140,2000,32000,32,1,54
+181,140,2000,32000,32,1,54
+32,140,2000,4000,8,1,20
+82,57,4000,16000,1,6,12
+171,57,4000,24000,64,12,16
+361,26,16000,32000,64,16,24
+350,26,16000,32000,64,8,24
+220,26,8000,32000,0,8,24
+113,26,8000,16000,0,8,16
+15,480,96,512,0,1,1
+21,203,1000,2000,0,1,5
+35,115,512,6000,16,1,6
+18,1100,512,1500,0,1,1
+20,1100,768,2000,0,1,1
+20,600,768,2000,0,1,1
+28,400,2000,4000,0,1,1
+45,400,4000,8000,0,1,1
+18,900,1000,1000,0,1,2
+17,900,512,1000,0,1,2
+26,900,1000,4000,4,1,2
+28,900,1000,4000,8,1,2
+28,900,2000,4000,0,3,6
+31,225,2000,4000,8,3,6
+31,225,2000,4000,8,3,6
+42,180,2000,8000,8,1,6
+76,185,2000,16000,16,1,6
+76,180,2000,16000,16,1,6
+26,225,1000,4000,2,3,6
+59,25,2000,12000,8,1,4
+65,25,2000,12000,16,3,5
+101,17,4000,16000,8,6,12
+116,17,4000,16000,32,6,12
+18,1500,768,1000,0,0,0
+20,1500,768,2000,0,0,0
+20,800,768,2000,0,0,0
+30,50,2000,4000,0,3,6
+44,50,2000,8000,8,3,6
+44,50,2000,8000,8,1,6
+82,50,2000,16000,24,1,6
+82,50,2000,16000,24,1,6
+128,50,8000,16000,48,1,10
+37,100,1000,8000,0,2,6
+46,100,1000,8000,24,2,6
+46,100,1000,8000,24,3,6
+80,50,2000,16000,12,3,16
+88,50,2000,16000,24,6,16
+88,50,2000,16000,24,6,16
+33,150,512,4000,0,8,128
+46,115,2000,8000,16,1,3
+29,115,2000,4000,2,1,5
+53,92,2000,8000,32,1,6
+53,92,2000,8000,32,1,6
+41,92,2000,8000,4,1,6
+86,75,4000,16000,16,1,6
+95,60,4000,16000,32,1,6
+107,60,2000,16000,64,5,8
+117,60,4000,16000,64,5,8
+119,50,4000,16000,64,5,10
+120,72,4000,16000,64,8,16
+48,72,2000,8000,16,6,8
+126,40,8000,16000,32,8,16
+266,40,8000,32000,64,8,24
+270,35,8000,32000,64,8,24
+426,38,16000,32000,128,16,32
+151,48,4000,24000,32,8,24
+267,38,8000,32000,64,8,24
+603,30,16000,32000,256,16,24
+19,112,1000,1000,0,1,4
+21,84,1000,2000,0,1,6
+26,56,1000,4000,0,1,6
+35,56,2000,6000,0,1,8
+41,56,2000,8000,0,1,8
+47,56,4000,8000,0,1,8
+62,56,4000,12000,0,1,8
+78,56,4000,16000,0,1,8
+80,38,4000,8000,32,16,32
+80,38,4000,8000,32,16,32
+142,38,8000,16000,64,4,8
+281,38,8000,24000,160,4,8
+190,38,4000,16000,128,16,32
+21,200,1000,2000,0,1,2
+25,200,1000,4000,0,1,4
+67,200,2000,8000,64,1,5
+24,250,512,4000,0,1,7
+24,250,512,4000,0,4,7
+64,250,1000,16000,1,1,8
+25,160,512,4000,2,1,5
+20,160,512,2000,2,3,8
+29,160,1000,4000,8,1,14
+43,160,1000,8000,16,1,14
+53,160,2000,8000,32,1,13
+19,240,512,1000,8,1,3
+22,240,512,2000,8,1,5
+31,105,2000,4000,8,3,8
+41,105,2000,6000,16,6,16
+47,105,2000,8000,16,4,14
+99,52,4000,16000,32,4,12
+67,70,4000,12000,8,6,8
+81,59,4000,12000,32,6,12
+149,59,8000,16000,64,12,24
+183,26,8000,24000,32,8,16
+275,26,8000,32000,64,12,16
+382,26,8000,32000,128,24,32
+56,116,2000,8000,32,5,28
+182,50,2000,32000,24,6,26
+227,50,2000,32000,48,26,52
+341,50,2000,32000,112,52,104
+360,50,4000,32000,112,52,104
+919,30,8000,64000,96,12,176
+978,30,8000,64000,128,12,176
+24,180,262,4000,0,1,3
+24,180,512,4000,0,1,3
+24,180,262,4000,0,1,3
+24,180,512,4000,0,1,3
+37,124,1000,8000,0,1,8
+50,98,1000,8000,32,2,8
+41,125,2000,8000,0,2,14
+47,480,512,8000,32,0,0
+25,480,1000,4000,0,0,0
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/examples/src/main/resources/datasets/knn/iris.txt
----------------------------------------------------------------------
diff --git a/examples/src/main/resources/datasets/knn/iris.txt b/examples/src/main/resources/datasets/knn/iris.txt
new file mode 100644
index 0000000..18f5f7c
--- /dev/null
+++ b/examples/src/main/resources/datasets/knn/iris.txt
@@ -0,0 +1,150 @@
+1.0 5.1 3.5 1.4 0.2
+1.0 4.9 3.0 1.4 0.2
+1.0 4.7 3.2 1.3 0.2
+1.0 4.6 3.1 1.5 0.2
+1.0 5.0 3.6 1.4 0.2
+1.0 5.4 3.9 1.7 0.4
+1.0 4.6 3.4 1.4 0.3
+1.0 5.0 3.4 1.5 0.2
+1.0 4.4 2.9 1.4 0.2
+1.0 4.9 3.1 1.5 0.1
+1.0 5.4 3.7 1.5 0.2
+1.0 4.8 3.4 1.6 0.2
+1.0 4.8 3.0 1.4 0.1
+1.0 4.3 3.0 1.1 0.1
+1.0 5.8 4.0 1.2 0.2
+1.0 5.7 4.4 1.5 0.4
+1.0 5.4 3.9 1.3 0.4
+1.0 5.1 3.5 1.4 0.3
+1.0 5.7 3.8 1.7 0.3
+1.0 5.1 3.8 1.5 0.3
+1.0 5.4 3.4 1.7 0.2
+1.0 5.1 3.7 1.5 0.4
+1.0 4.6 3.6 1.0 0.2
+1.0 5.1 3.3 1.7 0.5
+1.0 4.8 3.4 1.9 0.2
+1.0 5.0 3.0 1.6 0.2
+1.0 5.0 3.4 1.6 0.4
+1.0 5.2 3.5 1.5 0.2
+1.0 5.2 3.4 1.4 0.2
+1.0 4.7 3.2 1.6 0.2
+1.0 4.8 3.1 1.6 0.2
+1.0 5.4 3.4 1.5 0.4
+1.0 5.2 4.1 1.5 0.1
+1.0 5.5 4.2 1.4 0.2
+1.0 4.9 3.1 1.5 0.1
+1.0 5.0 3.2 1.2 0.2
+1.0 5.5 3.5 1.3 0.2
+1.0 4.9 3.1 1.5 0.1
+1.0 4.4 3.0 1.3 0.2
+1.0 5.1 3.4 1.5 0.2
+1.0 5.0 3.5 1.3 0.3
+1.0 4.5 2.3 1.3 0.3
+1.0 4.4 3.2 1.3 0.2
+1.0 5.0 3.5 1.6 0.6
+1.0 5.1 3.8 1.9 0.4
+1.0 4.8 3.0 1.4 0.3
+1.0 5.1 3.8 1.6 0.2
+1.0 4.6 3.2 1.4 0.2
+1.0 5.3 3.7 1.5 0.2
+1.0 5.0 3.3 1.4 0.2
+2.0 7.0 3.2 4.7 1.4
+2.0 6.4 3.2 4.5 1.5
+2.0 6.9 3.1 4.9 1.5
+2.0 5.5 2.3 4.0 1.3
+2.0 6.5 2.8 4.6 1.5
+2.0 5.7 2.8 4.5 1.3
+2.0 6.3 3.3 4.7 1.6
+2.0 4.9 2.4 3.3 1.0
+2.0 6.6 2.9 4.6 1.3
+2.0 5.2 2.7 3.9 1.4
+2.0 5.0 2.0 3.5 1.0
+2.0 5.9 3.0 4.2 1.5
+2.0 6.0 2.2 4.0 1.0
+2.0 6.1 2.9 4.7 1.4
+2.0 5.6 2.9 3.6 1.3
+2.0 6.7 3.1 4.4 1.4
+2.0 5.6 3.0 4.5 1.5
+2.0 5.8 2.7 4.1 1.0
+2.0 6.2 2.2 4.5 1.5
+2.0 5.6 2.5 3.9 1.1
+2.0 5.9 3.2 4.8 1.8
+2.0 6.1 2.8 4.0 1.3
+2.0 6.3 2.5 4.9 1.5
+2.0 6.1 2.8 4.7 1.2
+2.0 6.4 2.9 4.3 1.3
+2.0 6.6 3.0 4.4 1.4
+2.0 6.8 2.8 4.8 1.4
+2.0 6.7 3.0 5.0 1.7
+2.0 6.0 2.9 4.5 1.5
+2.0 5.7 2.6 3.5 1.0
+2.0 5.5 2.4 3.8 1.1
+2.0 5.5 2.4 3.7 1.0
+2.0 5.8 2.7 3.9 1.2
+2.0 6.0 2.7 5.1 1.6
+2.0 5.4 3.0 4.5 1.5
+2.0 6.0 3.4 4.5 1.6
+2.0 6.7 3.1 4.7 1.5
+2.0 6.3 2.3 4.4 1.3
+2.0 5.6 3.0 4.1 1.3
+2.0 5.5 2.5 4.0 1.3
+2.0 5.5 2.6 4.4 1.2
+2.0 6.1 3.0 4.6 1.4
+2.0 5.8 2.6 4.0 1.2
+2.0 5.0 2.3 3.3 1.0
+2.0 5.6 2.7 4.2 1.3
+2.0 5.7 3.0 4.2 1.2
+2.0 5.7 2.9 4.2 1.3
+2.0 6.2 2.9 4.3 1.3
+2.0 5.1 2.5 3.0 1.1
+2.0 5.7 2.8 4.1 1.3
+3.0 6.3 3.3 6.0 2.5
+3.0 5.8 2.7 5.1 1.9
+3.0 7.1 3.0 5.9 2.1
+3.0 6.3 2.9 5.6 1.8
+3.0 6.5 3.0 5.8 2.2
+3.0 7.6 3.0 6.6 2.1
+3.0 4.9 2.5 4.5 1.7
+3.0 7.3 2.9 6.3 1.8
+3.0 6.7 2.5 5.8 1.8
+3.0 7.2 3.6 6.1 2.5
+3.0 6.5 3.2 5.1 2.0
+3.0 6.4 2.7 5.3 1.9
+3.0 6.8 3.0 5.5 2.1
+3.0 5.7 2.5 5.0 2.0
+3.0 5.8 2.8 5.1 2.4
+3.0 6.4 3.2 5.3 2.3
+3.0 6.5 3.0 5.5 1.8
+3.0 7.7 3.8 6.7 2.2
+3.0 7.7 2.6 6.9 2.3
+3.0 6.0 2.2 5.0 1.5
+3.0 6.9 3.2 5.7 2.3
+3.0 5.6 2.8 4.9 2.0
+3.0 7.7 2.8 6.7 2.0
+3.0 6.3 2.7 4.9 1.8
+3.0 6.7 3.3 5.7 2.1
+3.0 7.2 3.2 6.0 1.8
+3.0 6.2 2.8 4.8 1.8
+3.0 6.1 3.0 4.9 1.8
+3.0 6.4 2.8 5.6 2.1
+3.0 7.2 3.0 5.8 1.6
+3.0 7.4 2.8 6.1 1.9
+3.0 7.9 3.8 6.4 2.0
+3.0 6.4 2.8 5.6 2.2
+3.0 6.3 2.8 5.1 1.5
+3.0 6.1 2.6 5.6 1.4
+3.0 7.7 3.0 6.1 2.3
+3.0 6.3 3.4 5.6 2.4
+3.0 6.4 3.1 5.5 1.8
+3.0 6.0 3.0 4.8 1.8
+3.0 6.9 3.1 5.4 2.1
+3.0 6.7 3.1 5.6 2.4
+3.0 6.9 3.1 5.1 2.3
+3.0 5.8 2.7 5.1 1.9
+3.0 6.8 3.2 5.9 2.3
+3.0 6.7 3.3 5.7 2.5
+3.0 6.7 3.0 5.2 2.3
+3.0 6.3 2.5 5.0 1.9
+3.0 6.5 3.0 5.2 2.0
+3.0 6.2 3.4 5.4 2.3
+3.0 5.9 3.0 5.1 1.8
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
index 81f7607..ee2f442 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDataset.java
@@ -270,6 +270,24 @@ public class LabeledDataset implements Serializable {
}
/**
+ * Returns new copy of labels of all labeled vectors NOTE: This method is useful for copying labels from test
+ * dataset.
+ *
+ * @return Copy of labels.
+ */
+ public double[] labels() {
+ assert data != null;
+ assert data.length > 0;
+
+ double[] labels = new double[data.length];
+
+ for (int i = 0; i < data.length; i++)
+ labels[i] = (double)data[i].label();
+
+ return labels;
+ }
+
+ /**
* Fill the label with given value.
*
* @param idx Index of observation.
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDatasetTestTrainPair.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDatasetTestTrainPair.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDatasetTestTrainPair.java
new file mode 100644
index 0000000..dd3d244
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/LabeledDatasetTestTrainPair.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.structures;
+
+import java.io.Serializable;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Class for splitting Labeled Dataset on train and test sets.
+ */
+public class LabeledDatasetTestTrainPair implements Serializable {
+ /** Data to keep train set. */
+ private LabeledDataset train;
+
+ /** Data to keep test set. */
+ private LabeledDataset test;
+
+ /**
+ * Creates two subsets of given dataset.
+ * <p>
+ * NOTE: This method uses next algorithm with O(n log n) by calculations and O(n) by memory.
+ * </p>
+ * @param dataset The dataset to split on train and test subsets.
+ * @param testPercentage The percentage of the test subset.
+ */
+ public LabeledDatasetTestTrainPair(LabeledDataset dataset, double testPercentage) {
+ assert testPercentage > 0.0;
+ assert testPercentage < 1.0;
+ final int datasetSize = dataset.rowSize();
+ assert datasetSize > 2;
+
+ final int testSize = (int)Math.floor(datasetSize * testPercentage);
+ final int trainSize = datasetSize - testSize;
+
+ final TreeSet<Integer> sortedTestIndices = getSortedIndices(datasetSize, testSize);
+
+
+ LabeledVector[] testVectors = new LabeledVector[testSize];
+ LabeledVector[] trainVectors = new LabeledVector[trainSize];
+
+
+ int datasetCntr = 0;
+ int trainCntr = 0;
+ int testCntr = 0;
+
+ for (Integer idx: sortedTestIndices){ // guarantee order as iterator
+ testVectors[testCntr] = dataset.getRow(idx);
+ testCntr++;
+
+ for (int i = datasetCntr; i < idx; i++) {
+ trainVectors[trainCntr] = dataset.getRow(i);
+ trainCntr++;
+ }
+
+ datasetCntr = idx + 1;
+ }
+ if(datasetCntr < datasetSize){
+ for (int i = datasetCntr; i < datasetSize; i++) {
+ trainVectors[trainCntr] = dataset.getRow(i);
+ trainCntr++;
+ }
+ }
+
+ test = new LabeledDataset(testVectors, testSize);
+ train = new LabeledDataset(trainVectors, trainSize);
+ }
+
+ /** This method generates "random double, integer" pairs, sort them, gets first "testSize" elements and returns appropriate indices */
+ @NotNull private TreeSet<Integer> getSortedIndices(int datasetSize, int testSize) {
+ Random rnd = new Random();
+ TreeMap<Double, Integer> randomIdxPairs = new TreeMap<>();
+ for (int i = 0; i < datasetSize; i++)
+ randomIdxPairs.put(rnd.nextDouble(), i);
+
+ final TreeMap<Double, Integer> testIdxPairs = randomIdxPairs.entrySet().stream()
+ .limit(testSize)
+ .collect(TreeMap::new, (m, e) -> m.put(e.getKey(), e.getValue()), Map::putAll);
+
+ return new TreeSet<>(testIdxPairs.values());
+ }
+
+ /**
+ * Train subset of the whole dataset.
+ * @return Train subset.
+ */
+ public LabeledDataset train() {
+ return train;
+ }
+
+ /**
+ * Test subset of the whole dataset.
+ * @return Test subset.
+ */
+ public LabeledDataset test() {
+ return test;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/da782958/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
index 32bd37b..c64a8d8 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
@@ -28,6 +28,7 @@ import org.apache.ignite.ml.math.exceptions.NoDataException;
import org.apache.ignite.ml.math.exceptions.knn.EmptyFileException;
import org.apache.ignite.ml.math.exceptions.knn.FileParsingException;
import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
import org.apache.ignite.ml.structures.LabeledVector;
/** Tests behaviour of KNNClassificationTest. */
@@ -205,4 +206,61 @@ public class LabeledDatasetTest extends BaseKNNTest {
assertEquals(training.features(2).get(1), 0.0);
}
+
+ /** */
+ public void testSplitting() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ double[][] mtx =
+ new double[][] {
+ {1.0, 1.0},
+ {1.0, 2.0},
+ {2.0, 1.0},
+ {-1.0, -1.0},
+ {-1.0, -2.0},
+ {-2.0, -1.0}};
+ double[] lbs = new double[] {1.0, 1.0, 1.0, 2.0, 2.0, 2.0};
+
+ LabeledDataset training = new LabeledDataset(mtx, lbs);
+
+ LabeledDatasetTestTrainPair split1 = new LabeledDatasetTestTrainPair(training, 0.67);
+
+ assertEquals(4, split1.test().rowSize());
+ assertEquals(2, split1.train().rowSize());
+
+ LabeledDatasetTestTrainPair split2 = new LabeledDatasetTestTrainPair(training, 0.65);
+
+ assertEquals(3, split2.test().rowSize());
+ assertEquals(3, split2.train().rowSize());
+
+ LabeledDatasetTestTrainPair split3 = new LabeledDatasetTestTrainPair(training, 0.4);
+
+ assertEquals(2, split3.test().rowSize());
+ assertEquals(4, split3.train().rowSize());
+
+ LabeledDatasetTestTrainPair split4 = new LabeledDatasetTestTrainPair(training, 0.3);
+
+ assertEquals(1, split4.test().rowSize());
+ assertEquals(5, split4.train().rowSize());
+ }
+
+ /** */
+ public void testLabels() {
+ IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+ double[][] mtx =
+ new double[][] {
+ {1.0, 1.0},
+ {1.0, 2.0},
+ {2.0, 1.0},
+ {-1.0, -1.0},
+ {-1.0, -2.0},
+ {-2.0, -1.0}};
+ double[] lbs = new double[] {1.0, 1.0, 1.0, 2.0, 2.0, 2.0};
+
+ LabeledDataset dataset = new LabeledDataset(mtx, lbs);
+ final double[] labels = dataset.labels();
+ for (int i = 0; i < lbs.length; i++)
+ assertEquals(lbs[i], labels[i]);
+ }
}
[14/50] [abbrv] ignite git commit: IGNITE-7199 Web console: several
minor improvements.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/public/images/icons/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/images/icons/index.js b/modules/web-console/frontend/public/images/icons/index.js
index 2d0f649..48d9227 100644
--- a/modules/web-console/frontend/public/images/icons/index.js
+++ b/modules/web-console/frontend/public/images/icons/index.js
@@ -26,5 +26,7 @@ export search from './search.svg';
export refresh from './refresh.svg';
export sort from './sort.svg';
export info from './info.svg';
-export connectedClusters from './connectedClusters.svg';
+export checkmark from './checkmark.svg';
+export alert from './alert.svg';
export attention from './attention.svg';
+export connectedClusters from './connectedClusters.svg';
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/public/images/icons/sort.svg
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/images/icons/sort.svg b/modules/web-console/frontend/public/images/icons/sort.svg
index 7e4bb52..8195c79 100644
--- a/modules/web-console/frontend/public/images/icons/sort.svg
+++ b/modules/web-console/frontend/public/images/icons/sort.svg
@@ -1 +1 @@
-<svg id="drag-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill="currentColor" d="M 4,6.9 H 6.3 V 4.6 H 4 Z m 4.6,0 h 2.3 V 4.6 H 8.6 Z M 4,11.4 H 6.3 V 9.1 H 4 Z m 4.6,0 h 2.3 V 9.1 H 8.6 Z M 4,16 H 6.3 V 13.7 H 4 Z m 4.6,0 h 2.3 V 13.7 H 8.6 Z M 4,2.3 H 6.3 V 0 H 4 Z m 4.6,0 h 2.3 V 0 H 8.6 Z"/></svg>
\ No newline at end of file
+<svg id="drag-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path fill="currentColor" d="M 4,6.9 H 6.3 V 4.6 H 4 Z m 4.6,0 h 2.3 V 4.6 H 8.6 Z M 4,11.4 H 6.3 V 9.1 H 4 Z m 4.6,0 h 2.3 V 9.1 H 8.6 Z M 4,16 H 6.3 V 13.7 H 4 Z m 4.6,0 h 2.3 V 13.7 H 8.6 Z M 4,2.3 H 6.3 V 0 H 4 Z m 4.6,0 h 2.3 V 0 H 8.6 Z"/></svg>
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/views/includes/header-left.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/includes/header-left.pug b/modules/web-console/frontend/views/includes/header-left.pug
index 6b7fe6a..42d4dcb 100644
--- a/modules/web-console/frontend/views/includes/header-left.pug
+++ b/modules/web-console/frontend/views/includes/header-left.pug
@@ -18,7 +18,7 @@
a(ui-sref='base.configuration.tabs' ui-sref-active='active')
| Configure
-.wch-nav-item(ng-controller='notebookController')
+.wch-nav-item(ng-if='!$root.user.becomeUsed' ng-controller='notebookController')
a(ng-if='IgniteDemoMode' ui-sref='base.sql.demo' ui-sref-active='active')
| Queries
@@ -41,7 +41,7 @@
.wch-content(ignite-navbar)
.wch-nav-item(ng-repeat='item in navbar.items')
- div(ng-if='item.children' ng-click='$event.stopPropagation()'
+ div(ng-if='$root.user.becomeUsed ? item.canBecomed : item.children' ng-click='$event.stopPropagation()'
ng-class='{active: $state.includes(item.sref)}'
bs-dropdown='item.children'
data-placement='bottom-left'
@@ -53,5 +53,5 @@
span {{::item.text}}
span.caret
- a(ng-if='!item.children' ui-sref='{{item.sref}}' ui-sref-active='active')
- | {{::item.text}}
\ No newline at end of file
+ a(ng-if='$root.user.becomeUsed ? item.canBecomed : !item.children' ui-sref='{{item.sref}}' ui-sref-active='active')
+ | {{::item.text}}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/views/includes/header-right.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/includes/header-right.pug b/modules/web-console/frontend/views/includes/header-right.pug
index 56fd102..b223f2e 100644
--- a/modules/web-console/frontend/views/includes/header-right.pug
+++ b/modules/web-console/frontend/views/includes/header-right.pug
@@ -14,12 +14,14 @@
See the License for the specific language governing permissions and
limitations under the License.
-.wch-demo-toggle(ng-controller='demoController')
+.wch-demo-toggle(ng-if='!$root.user.becomeUsed' ng-controller='demoController')
button.btn-ignite.btn-ignite--success(
ng-if='!IgniteDemoMode'
ng-click='startDemo()'
) Start Demo
+web-console-header-extension
+
.wch-nav-item(ignite-userbar)
div(
ng-class='{active: $state.includes("base.settings")}'
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/views/settings/profile.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/views/settings/profile.tpl.pug b/modules/web-console/frontend/views/settings/profile.tpl.pug
index 54c8105..0dc35e4 100644
--- a/modules/web-console/frontend/views/settings/profile.tpl.pug
+++ b/modules/web-console/frontend/views/settings/profile.tpl.pug
@@ -17,6 +17,9 @@
mixin lbl(txt)
label.col-sm-2.required.labelFormField #{txt}
+mixin lbl-not-required(txt)
+ label.col-sm-2.labelFormField #{txt}
+
.row(ng-controller='profileController')
.docs-content
.docs-header
@@ -36,7 +39,11 @@ mixin lbl(txt)
.details-row
+lbl('Email:')
.col-xs-5.col-sm-4
- input#profile-email.form-control(ignite-on-enter-focus-move='profile-company' type='email' ng-model='user.email' placeholder='Input email' required)
+ input#profile-email.form-control(ignite-on-enter-focus-move='profile-phone' type='email' ng-model='user.email' placeholder='Input email' required)
+ .details-row
+ +lbl-not-required('Phone:')
+ .col-xs-5.col-sm-4
+ input#profile-phone.form-control(ignite-on-enter-focus-move='profile-company' type='tel' ng-model='user.phone' placeholder='Input phone')
.details-row
+lbl('Company:')
.col-xs-5.col-sm-4
[25/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk-ce
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk-ce
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2b74163c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2b74163c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2b74163c
Branch: refs/heads/ignite-zk
Commit: 2b74163c0a06767683a8d7c784ca94fc469c6b72
Parents: cbe9980 061ec6a
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 11:03:47 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 11:03:47 2017 +0300
----------------------------------------------------------------------
bin/control.bat | 1 +
bin/control.sh | 1 +
.../ml/clustering/FuzzyCMeansExample.java | 4 +-
.../KMeansDistributedClustererExample.java | 2 +-
.../clustering/KMeansLocalClustererExample.java | 4 +-
.../KNNClassificationExample.java | 151 +++++
.../ml/knn/classification/package-info.java | 22 +
.../ignite/examples/ml/knn/package-info.java | 22 +
.../ml/knn/regression/KNNRegressionExample.java | 152 +++++
.../ml/knn/regression/package-info.java | 22 +
.../src/main/resources/datasets/knn/README.md | 2 +
.../resources/datasets/knn/cleared_machines.txt | 209 ++++++
.../src/main/resources/datasets/knn/iris.txt | 150 +++++
.../spi/checkpoint/s3/S3CheckpointSpi.java | 46 +-
.../spi/checkpoint/s3/S3CheckpointSpiMBean.java | 6 +
.../tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java | 39 +-
...ckpointSpiStartStopSSEAlgorithmSelfTest.java | 49 ++
.../TcpDiscoveryS3IpFinderAbstractSelfTest.java | 22 +-
...scoveryS3IpFinderBucketEndpointSelfTest.java | 16 +-
...DiscoveryS3IpFinderSSEAlgorithmSelfTest.java | 48 ++
.../ignite/testsuites/IgniteS3TestSuite.java | 4 +
.../configuration/MemoryConfiguration.java | 2 +-
.../ignite/internal/ClusterMetricsSnapshot.java | 4 +-
.../apache/ignite/internal/IgniteKernal.java | 2 +-
.../internal/commandline/CommandHandler.java | 2 +-
.../pagemem/impl/PageMemoryNoStoreImpl.java | 2 +-
.../internal/pagemem/store/PageStore.java | 5 +-
.../processors/affinity/AffinityAssignment.java | 5 +
.../affinity/GridAffinityAssignment.java | 45 +-
.../affinity/HistoryAffinityAssignment.java | 15 +
.../cache/CacheAffinitySharedManager.java | 31 +-
.../processors/cache/GridCacheTtlManager.java | 3 +-
.../processors/cache/GridCacheUtils.java | 6 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 41 +-
.../dht/GridDhtPartitionTopologyImpl.java | 475 +++++++-------
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../dht/GridPartitionedGetFuture.java | 2 +-
.../dht/GridPartitionedSingleGetFuture.java | 3 +-
.../GridDhtPartitionsExchangeFuture.java | 14 +-
.../distributed/near/GridNearGetFuture.java | 2 +-
.../local/atomic/GridLocalAtomicCache.java | 171 ++---
.../GridCacheDatabaseSharedManager.java | 9 +-
.../persistence/GridCacheOffheapManager.java | 4 +
.../processors/cache/persistence/RowStore.java | 2 +
.../cache/persistence/file/FilePageStore.java | 32 +-
.../persistence/file/FilePageStoreManager.java | 7 +-
.../persistence/pagemem/PageMemoryImpl.java | 58 +-
.../persistence/pagemem/PagesWriteThrottle.java | 3 +-
.../cache/persistence/tree/BPlusTree.java | 10 +-
.../processors/cache/tree/CacheDataTree.java | 2 +
.../cache/tree/PendingEntriesTree.java | 2 +
.../schema/SchemaIndexCacheVisitorImpl.java | 2 +-
.../internal/util/OffheapReadWriteLock.java | 8 +-
.../internal/util/lang/GridTreePrinter.java | 19 +-
.../cache/VisorCacheEvictionConfiguration.java | 4 +-
.../cache/VisorCacheNearConfiguration.java | 8 +-
.../internal/visor/util/VisorTaskUtils.java | 14 +-
.../pagemem/impl/PageMemoryNoLoadSelfTest.java | 10 +-
.../cache/persistence/DummyPageIO.java | 41 ++
.../IgnitePdsContinuousRestartTest.java | 5 -
...gnitePdsRecoveryAfterFileCorruptionTest.java | 83 ++-
...ckpointSimulationWithRealCpDisabledTest.java | 53 +-
.../db/file/IgnitePdsEvictionTest.java | 30 +
.../query/h2/DmlStatementsProcessor.java | 637 ++----------------
.../processors/query/h2/IgniteH2Indexing.java | 69 +-
.../processors/query/h2/UpdateResult.java | 4 +-
.../processors/query/h2/database/H2Tree.java | 6 +-
.../query/h2/database/H2TreeIndex.java | 220 ++++---
.../processors/query/h2/dml/DmlAstUtils.java | 609 ++++++++++++++++++
.../processors/query/h2/dml/DmlBatchSender.java | 232 +++++++
.../query/h2/dml/DmlDistributedPlanInfo.java | 56 ++
.../h2/dml/DmlPageProcessingErrorResult.java | 76 +++
.../query/h2/dml/DmlPageProcessingResult.java | 68 ++
.../processors/query/h2/dml/DmlUtils.java | 118 ++++
.../processors/query/h2/dml/FastUpdate.java | 175 +++++
.../query/h2/dml/FastUpdateArguments.java | 53 --
.../processors/query/h2/dml/UpdatePlan.java | 389 ++++++++---
.../query/h2/dml/UpdatePlanBuilder.java | 82 ++-
.../processors/query/h2/sql/DmlAstUtils.java | 644 -------------------
.../query/h2/sql/GridSqlQueryParser.java | 1 +
.../h2/twostep/GridReduceQueryExecutor.java | 44 +-
.../cache/GridCacheFullTextQuerySelfTest.java | 30 +-
.../IgniteCacheAbstractFieldsQuerySelfTest.java | 43 +-
...ldsQueryJoinNoPrimaryPartitionsSelfTest.java | 151 +++++
.../cache/index/H2ConnectionLeaksSelfTest.java | 169 +++++
.../IgniteCacheQuerySelfTestSuite.java | 4 +
.../ignite/ml/FuzzyCMeansModelFormat.java | 2 +-
.../org/apache/ignite/ml/KMeansModelFormat.java | 2 +-
.../ml/clustering/BaseFuzzyCMeansClusterer.java | 2 +-
.../ml/clustering/BaseKMeansClusterer.java | 2 +-
.../FuzzyCMeansDistributedClusterer.java | 2 +-
.../clustering/FuzzyCMeansLocalClusterer.java | 2 +-
.../ignite/ml/clustering/FuzzyCMeansModel.java | 2 +-
.../clustering/KMeansDistributedClusterer.java | 2 +-
.../ml/clustering/KMeansLocalClusterer.java | 2 +-
.../ignite/ml/clustering/KMeansModel.java | 2 +-
.../apache/ignite/ml/knn/models/KNNModel.java | 233 +++++++
.../ignite/ml/knn/models/KNNModelFormat.java | 92 +++
.../ignite/ml/knn/models/KNNStrategy.java | 27 +
.../ignite/ml/knn/models/Normalization.java | 32 +
.../ignite/ml/knn/models/package-info.java | 22 +
.../org/apache/ignite/ml/knn/package-info.java | 22 +
.../regression/KNNMultipleLinearRegression.java | 83 +++
.../ignite/ml/knn/regression/package-info.java | 22 +
.../apache/ignite/ml/math/DistanceMeasure.java | 38 --
.../ignite/ml/math/EuclideanDistance.java | 58 --
.../ml/math/distances/DistanceMeasure.java | 39 ++
.../ml/math/distances/EuclideanDistance.java | 59 ++
.../ml/math/distances/HammingDistance.java | 65 ++
.../ml/math/distances/ManhattanDistance.java | 59 ++
.../ignite/ml/math/distances/package-info.java | 22 +
.../ignite/ml/math/distributed/CacheUtils.java | 2 +-
.../distributed/keys/impl/SparseMatrixKey.java | 1 +
.../math/exceptions/knn/EmptyFileException.java | 37 ++
.../exceptions/knn/FileParsingException.java | 39 ++
.../exceptions/knn/NoLabelVectorException.java | 37 ++
.../knn/SmallTrainingDatasetSizeException.java | 38 ++
.../ml/math/exceptions/knn/package-info.java | 22 +
.../math/functions/IgniteToDoubleFunction.java | 6 +-
.../ignite/ml/structures/LabeledDataset.java | 475 ++++++++++++++
.../structures/LabeledDatasetTestTrainPair.java | 116 ++++
.../ignite/ml/structures/LabeledVector.java | 37 +-
.../org/apache/ignite/ml/IgniteMLTestSuite.java | 2 +
.../org/apache/ignite/ml/LocalModelsTest.java | 39 +-
.../FuzzyCMeansDistributedClustererTest.java | 4 +-
.../FuzzyCMeansLocalClustererTest.java | 4 +-
...KMeansDistributedClustererTestMultiNode.java | 2 +-
...MeansDistributedClustererTestSingleNode.java | 4 +-
.../ml/clustering/KMeansLocalClustererTest.java | 2 +-
.../org/apache/ignite/ml/knn/BaseKNNTest.java | 91 +++
.../ignite/ml/knn/KNNClassificationTest.java | 153 +++++
.../ml/knn/KNNMultipleLinearRegressionTest.java | 157 +++++
.../org/apache/ignite/ml/knn/KNNTestSuite.java | 33 +
.../ignite/ml/knn/LabeledDatasetTest.java | 266 ++++++++
.../ignite/ml/math/MathImplLocalTestSuite.java | 4 +-
.../ignite/ml/math/distances/DistanceTest.java | 75 +++
.../OLSMultipleLinearRegressionTest.java | 1 +
.../ml/trees/ColumnDecisionTreeTrainerTest.java | 6 +-
.../ColumnDecisionTreeTrainerBenchmark.java | 6 +-
.../ml/src/test/resources/datasets/README.md | 2 +
.../resources/datasets/knn/cleared_machines.txt | 209 ++++++
.../src/test/resources/datasets/knn/empty.txt | 0
.../ml/src/test/resources/datasets/knn/iris.txt | 150 +++++
.../resources/datasets/knn/iris_incorrect.txt | 150 +++++
.../resources/datasets/knn/machine.data.txt | 209 ++++++
.../test/resources/datasets/knn/missed_data.txt | 3 +
.../src/test/resources/datasets/knn/no_data.txt | 6 +
.../include/ignite/common/platform_utils.h | 20 +-
.../os/linux/src/common/platform_utils.cpp | 34 +-
.../common/os/win/src/common/platform_utils.cpp | 29 +-
.../platforms/cpp/core-test/src/test_utils.cpp | 8 +-
modules/platforms/cpp/core/src/ignition.cpp | 10 +-
.../cpp/jni/include/ignite/jni/utils.h | 17 +-
.../platforms/cpp/jni/os/linux/src/utils.cpp | 120 ++--
modules/platforms/cpp/jni/os/win/src/utils.cpp | 133 ++--
.../platforms/cpp/odbc-test/src/test_utils.cpp | 8 +-
.../ignite/visor/commands/VisorConsole.scala | 2 +-
.../commands/common/VisorConsoleCommand.scala | 5 +-
.../commands/tasks/VisorTasksCommand.scala | 34 +-
.../scala/org/apache/ignite/visor/visor.scala | 3 +-
modules/web-console/DEVNOTES.txt | 2 +-
modules/web-console/backend/.eslintrc | 185 ------
modules/web-console/backend/app/agentSocket.js | 8 +-
.../web-console/backend/app/agentsHandler.js | 35 +-
modules/web-console/backend/app/apiServer.js | 9 +-
.../web-console/backend/app/browsersHandler.js | 15 +-
modules/web-console/backend/app/configure.js | 14 +-
modules/web-console/backend/app/mongo.js | 14 +-
modules/web-console/backend/app/mongoose.js | 8 +-
modules/web-console/backend/app/nconf.js | 6 +-
modules/web-console/backend/app/settings.js | 6 +-
.../backend/config/settings.json.sample | 8 +-
.../1508395969410-init-registered-date.js | 33 +
modules/web-console/backend/package.json | 2 +
.../web-console/backend/routes/activities.js | 37 +-
modules/web-console/backend/routes/admin.js | 8 +-
modules/web-console/backend/routes/caches.js | 6 +-
modules/web-console/backend/routes/clusters.js | 6 +-
.../web-console/backend/routes/configuration.js | 6 +-
modules/web-console/backend/routes/demo.js | 7 +-
modules/web-console/backend/routes/domains.js | 6 +-
modules/web-console/backend/routes/downloads.js | 6 +-
modules/web-console/backend/routes/igfss.js | 6 +-
modules/web-console/backend/routes/notebooks.js | 7 +-
modules/web-console/backend/routes/profile.js | 10 +-
modules/web-console/backend/routes/public.js | 9 +-
modules/web-console/backend/services/Utils.js | 51 ++
.../web-console/backend/services/activities.js | 7 +-
modules/web-console/backend/services/auth.js | 27 +-
modules/web-console/backend/services/caches.js | 11 +-
.../web-console/backend/services/clusters.js | 11 +-
.../backend/services/configurations.js | 5 +-
modules/web-console/backend/services/domains.js | 11 +-
.../web-console/backend/services/downloads.js | 13 +-
modules/web-console/backend/services/igfss.js | 11 +-
modules/web-console/backend/services/mails.js | 89 +--
.../web-console/backend/services/notebooks.js | 11 +-
.../backend/services/notifications.js | 5 +-
.../web-console/backend/services/sessions.js | 5 +-
modules/web-console/backend/services/users.js | 26 +-
modules/web-console/frontend/app/app.js | 8 +
.../app/components/bs-select-menu/style.scss | 10 +-
.../app/components/bs-select-menu/template.pug | 15 +-
.../components/connected-clusters/controller.js | 36 ++
.../app/components/connected-clusters/index.js | 29 +
.../components/connected-clusters/style.scss | 40 ++
.../components/connected-clusters/template.pug | 18 +
.../app/components/grid-export/component.js | 52 ++
.../app/components/grid-export/index.js | 24 +
.../app/components/grid-export/template.pug | 18 +
.../app/components/grid-no-data/component.js | 33 +
.../app/components/grid-no-data/controller.js | 50 ++
.../app/components/grid-no-data/index.js | 24 +
.../app/components/grid-no-data/style.scss | 31 +
.../list-editable-cols/cols.template.pug | 1 -
.../list-editable-cols/row.directive.js | 2 +-
.../app/components/list-editable/style.scss | 12 +-
.../app/components/list-editable/template.pug | 6 +-
.../list-of-registered-users.column-defs.js | 49 +-
.../list-of-registered-users.tpl.pug | 2 +-
.../app/components/page-queries/controller.js | 5 +-
.../app/components/page-queries/style.scss | 55 ++
.../components/page-queries/template.tpl.pug | 27 +-
.../app/components/ui-grid-filters/directive.js | 62 ++
.../app/components/ui-grid-filters/index.js | 43 ++
.../app/components/ui-grid-filters/style.scss | 36 ++
.../app/components/ui-grid-filters/template.pug | 47 ++
.../components/web-console-header/component.js | 11 +
.../web-console-header-extension/component.js | 22 +
.../web-console-header-extension/template.pug | 15 +
.../app/components/web-console-header/index.js | 4 +-
.../components/web-console-header/style.scss | 2 +-
.../components/web-console-header/template.pug | 8 +-
.../app/directives/information/information.pug | 2 +-
.../app/directives/information/information.scss | 5 +-
.../frontend/app/filters/duration.filter.js | 2 +-
.../app/modules/agent/AgentManager.service.js | 6 +
.../generator/AbstractTransformer.js | 8 +-
.../generator/ConfigurationGenerator.js | 66 +-
.../generator/PlatformGenerator.js | 6 +-
.../generator/defaults/Cache.service.js | 14 +-
.../app/modules/form/field/input/text.scss | 1 +
.../frontend/app/modules/user/permissions.js | 4 +-
.../frontend/app/primitives/btn/index.scss | 17 +
.../app/primitives/form-field/index.scss | 2 +-
.../app/primitives/ui-grid-settings/index.scss | 12 +
.../frontend/app/primitives/ui-grid/index.scss | 2 +
.../frontend/app/services/Version.service.js | 4 +
modules/web-console/frontend/package.json | 3 +-
.../frontend/public/images/checkbox-active.svg | 25 +
.../frontend/public/images/checkbox.svg | 22 +
.../frontend/public/images/icons/alert.svg | 1 +
.../frontend/public/images/icons/attention.svg | 3 +
.../frontend/public/images/icons/checkmark.svg | 3 +
.../public/images/icons/connectedClusters.svg | 1 +
.../frontend/public/images/icons/index.js | 4 +
.../frontend/public/images/icons/sort.svg | 2 +-
.../frontend/views/includes/header-left.pug | 8 +-
.../frontend/views/includes/header-right.pug | 4 +-
.../frontend/views/settings/profile.tpl.pug | 9 +-
modules/web-console/pom.xml | 111 +++-
modules/yardstick/DEVNOTES-standalone.txt | 11 +-
modules/yardstick/DEVNOTES.txt | 11 +
.../yardstick/config/benchmark-ml.properties | 90 +++
modules/yardstick/pom-standalone.xml | 58 +-
modules/yardstick/pom.xml | 56 ++
.../apache/ignite/yardstick/ml/DataChanger.java | 65 ++
...niteKMeansDistributedClustererBenchmark.java | 75 +++
.../IgniteKMeansLocalClustererBenchmark.java | 50 ++
.../yardstick/ml/clustering/package-info.java | 22 +
.../IgniteCholeskyDecompositionBenchmark.java | 69 ++
.../IgniteEigenDecompositionBenchmark.java | 69 ++
.../IgniteLUDecompositionBenchmark.java | 75 +++
...niteSingularValueDecompositionBenchmark.java | 59 ++
.../ml/decomposition/package-info.java | 22 +
.../math/IgniteAbstractMatrixMulBenchmark.java | 107 +++
...niteDenseLocalOffHeapMatrixMulBenchmark.java | 32 +
...gniteDenseLocalOnHeapMatrixMulBenchmark.java | 32 +
...parseBlockDistributedMatrixMulBenchmark.java | 32 +
...niteSparseDistributedMatrixMulBenchmark.java | 35 +
.../IgniteSparseLocalMatrixMulBenchmark.java | 32 +
.../ignite/yardstick/ml/math/package-info.java | 22 +
.../ignite/yardstick/ml/package-info.java | 22 +
...iteOLSMultipleLinearRegressionBenchmark.java | 92 +++
.../yardstick/ml/regression/package-info.java | 22 +
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 ++
...niteColumnDecisionTreeVarianceBenchmark.java | 71 ++
.../yardstick/ml/trees/SplitDataGenerator.java | 426 ++++++++++++
.../ignite/yardstick/ml/trees/package-info.java | 22 +
parent/pom.xml | 1 +
290 files changed, 10617 insertions(+), 2816 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2b74163c/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2b74163c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2b74163c/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2b74163c/modules/yardstick/pom-standalone.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2b74163c/modules/yardstick/pom.xml
----------------------------------------------------------------------
[09/50] [abbrv] ignite git commit: IGNITE-4454 Minor fix for duration
filter.
Posted by sb...@apache.org.
IGNITE-4454 Minor fix for duration filter.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9f7bc54e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9f7bc54e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9f7bc54e
Branch: refs/heads/ignite-zk
Commit: 9f7bc54e53b42defada1602700da8a4c1abcd223
Parents: e24d4d0
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Wed Dec 13 23:02:40 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Wed Dec 13 23:02:40 2017 +0700
----------------------------------------------------------------------
modules/web-console/frontend/app/filters/duration.filter.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9f7bc54e/modules/web-console/frontend/app/filters/duration.filter.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/filters/duration.filter.js b/modules/web-console/frontend/app/filters/duration.filter.js
index 703b6a4..55ec7e0 100644
--- a/modules/web-console/frontend/app/filters/duration.filter.js
+++ b/modules/web-console/frontend/app/filters/duration.filter.js
@@ -36,6 +36,6 @@ export default [() => {
const s = Math.floor((t - d * cd - h * ch - m * cm) / cs);
const ms = Math.round(t % 1000);
- return a(d, 'd') + a(h, 'h') + a(m, 'm') + a(s, 's') + (t === 0 || (t < cm && ms !== 0) ? ms + 'ms' : '');
+ return a(d, 'd') + a(h, 'h') + a(m, 'm') + a(s, 's') + (t < 1000 || (t < cm && ms !== 0) ? ms + 'ms' : '');
};
}];
[21/50] [abbrv] ignite git commit: IGNITE-7200: SQL: simplified DML
module structure and restored encapsulation. This closes #3225.
Posted by sb...@apache.org.
IGNITE-7200: SQL: simplified DML module structure and restored encapsulation. This closes #3225.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/03bb5513
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/03bb5513
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/03bb5513
Branch: refs/heads/ignite-zk
Commit: 03bb551382be8303a4bcaf0afc3ffa0f9c2885dd
Parents: 802a166
Author: devozerov <vo...@gridgain.com>
Authored: Thu Dec 14 15:59:37 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Thu Dec 14 15:59:37 2017 +0300
----------------------------------------------------------------------
.../query/h2/DmlStatementsProcessor.java | 637 ++----------------
.../processors/query/h2/UpdateResult.java | 4 +-
.../processors/query/h2/dml/DmlAstUtils.java | 609 ++++++++++++++++++
.../processors/query/h2/dml/DmlBatchSender.java | 232 +++++++
.../query/h2/dml/DmlDistributedPlanInfo.java | 56 ++
.../h2/dml/DmlPageProcessingErrorResult.java | 76 +++
.../query/h2/dml/DmlPageProcessingResult.java | 68 ++
.../processors/query/h2/dml/DmlUtils.java | 118 ++++
.../processors/query/h2/dml/FastUpdate.java | 175 +++++
.../query/h2/dml/FastUpdateArguments.java | 53 --
.../processors/query/h2/dml/UpdatePlan.java | 389 ++++++++---
.../query/h2/dml/UpdatePlanBuilder.java | 82 ++-
.../processors/query/h2/sql/DmlAstUtils.java | 644 -------------------
.../query/h2/sql/GridSqlQueryParser.java | 1 +
14 files changed, 1752 insertions(+), 1392 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index c3d48dd..243d1dc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -17,42 +17,29 @@
package org.apache.ignite.internal.processors.query.h2;
-import java.lang.reflect.Array;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
-import java.sql.Time;
-import java.sql.Timestamp;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import javax.cache.processor.EntryProcessor;
import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
import javax.cache.processor.MutableEntry;
-import org.apache.ignite.IgniteCache;
+
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.binary.BinaryObject;
-import org.apache.ignite.binary.BinaryObjectBuilder;
import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheOperationContext;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
@@ -61,21 +48,19 @@ import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator
import org.apache.ignite.internal.processors.query.GridQueryCancel;
import org.apache.ignite.internal.processors.query.GridQueryFieldsResult;
import org.apache.ignite.internal.processors.query.GridQueryFieldsResultAdapter;
-import org.apache.ignite.internal.processors.query.GridQueryProperty;
-import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
-import org.apache.ignite.internal.processors.query.QueryUtils;
-import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArguments;
+import org.apache.ignite.internal.processors.query.h2.dml.DmlBatchSender;
+import org.apache.ignite.internal.processors.query.h2.dml.DmlDistributedPlanInfo;
+import org.apache.ignite.internal.processors.query.h2.dml.FastUpdate;
import org.apache.ignite.internal.processors.query.h2.dml.UpdateMode;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder;
-import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
import org.apache.ignite.internal.util.lang.IgniteSingletonIterator;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteInClosure;
@@ -85,19 +70,10 @@ import org.h2.command.dml.Delete;
import org.h2.command.dml.Insert;
import org.h2.command.dml.Merge;
import org.h2.command.dml.Update;
-import org.h2.table.Column;
-import org.h2.util.DateTimeUtils;
-import org.h2.util.LocalDateTimeUtils;
-import org.h2.value.Value;
-import org.h2.value.ValueDate;
-import org.h2.value.ValueTime;
-import org.h2.value.ValueTimestamp;
-import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.createJdbcSqlException;
import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.UPDATE_RESULT_META;
-import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT;
/**
*
@@ -142,7 +118,7 @@ public class DmlStatementsProcessor {
while (iter.hasNext()) {
UpdatePlan plan = iter.next().getValue();
- if (F.eq(cacheName, plan.tbl.cacheName()))
+ if (F.eq(cacheName, plan.cacheContext().name()))
iter.remove();
}
}
@@ -169,7 +145,7 @@ public class DmlStatementsProcessor {
UpdatePlan plan = getPlanForStatement(schemaName, conn, prepared, fieldsQry, loc, null);
- GridCacheContext<?, ?> cctx = plan.tbl.rowDescriptor().context();
+ GridCacheContext<?, ?> cctx = plan.cacheContext();
for (int i = 0; i < DFLT_DML_RERUN_ATTEMPTS; i++) {
CacheOperationContext opCtx = cctx.operationContextPerCall();
@@ -281,20 +257,20 @@ public class DmlStatementsProcessor {
UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, true, idx, null, null, null);
- if (!F.eq(streamer.cacheName(), plan.tbl.rowDescriptor().context().name()))
+ if (!F.eq(streamer.cacheName(), plan.cacheContext().name()))
throw new IgniteSQLException("Cross cache streaming is not supported, please specify cache explicitly" +
" in connection options", IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
- if (plan.mode == UpdateMode.INSERT && plan.rowsNum > 0) {
- assert plan.isLocSubqry;
+ if (plan.mode() == UpdateMode.INSERT && plan.rowCount() > 0) {
+ assert plan.isLocalSubquery();
- final GridCacheContext cctx = plan.tbl.rowDescriptor().context();
+ final GridCacheContext cctx = plan.cacheContext();
QueryCursorImpl<List<?>> cur;
- final ArrayList<List<?>> data = new ArrayList<>(plan.rowsNum);
+ final ArrayList<List<?>> data = new ArrayList<>(plan.rowCount());
- final GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()), plan.selectQry,
+ final GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()), plan.selectQuery(),
F.asList(args), null, false, 0, null);
QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
@@ -317,18 +293,18 @@ public class DmlStatementsProcessor {
}
}, null);
- if (plan.rowsNum == 1) {
- IgniteBiTuple t = rowToKeyValue(cctx, cur.iterator().next(), plan);
+ if (plan.rowCount() == 1) {
+ IgniteBiTuple t = plan.processRow(cur.iterator().next());
streamer.addData(t.getKey(), t.getValue());
return 1;
}
- Map<Object, Object> rows = new LinkedHashMap<>(plan.rowsNum);
+ Map<Object, Object> rows = new LinkedHashMap<>(plan.rowCount());
for (List<?> row : cur) {
- final IgniteBiTuple t = rowToKeyValue(cctx, row, plan);
+ final IgniteBiTuple t = plan.processRow(row);
rows.put(t.getKey(), t.getValue());
}
@@ -367,13 +343,15 @@ public class DmlStatementsProcessor {
UpdatePlan plan = getPlanForStatement(schemaName, c, prepared, fieldsQry, loc, errKeysPos);
- if (plan.fastUpdateArgs != null) {
+ FastUpdate fastUpdate = plan.fastUpdate();
+
+ if (fastUpdate != null) {
assert F.isEmpty(failedKeys) && errKeysPos == null;
- return doFastUpdate(plan, fieldsQry.getArgs());
+ return fastUpdate.execute(plan.cacheContext().cache(), fieldsQry.getArgs());
}
- if (plan.distributed != null) {
+ if (plan.distributedPlan() != null) {
UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel);
// null is returned in case not all nodes support distributed DML.
@@ -381,14 +359,14 @@ public class DmlStatementsProcessor {
return result;
}
- assert !F.isEmpty(plan.selectQry);
+ assert !F.isEmpty(plan.selectQuery());
QueryCursorImpl<List<?>> cur;
// Do a two-step query only if locality flag is not set AND if plan's SELECT corresponds to an actual
// sub-query and not some dummy stuff like "select 1, 2, 3;"
- if (!loc && !plan.isLocSubqry) {
- SqlFieldsQuery newFieldsQry = new SqlFieldsQuery(plan.selectQry, fieldsQry.isCollocated())
+ if (!loc && !plan.isLocalSubquery()) {
+ SqlFieldsQuery newFieldsQry = new SqlFieldsQuery(plan.selectQuery(), fieldsQry.isCollocated())
.setArgs(fieldsQry.getArgs())
.setDistributedJoins(fieldsQry.isDistributedJoins())
.setEnforceJoinOrder(fieldsQry.isEnforceJoinOrder())
@@ -400,7 +378,7 @@ public class DmlStatementsProcessor {
cancel, mainCacheId, true).get(0);
}
else {
- final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQry,
+ final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQuery(),
F.asList(fieldsQry.getArgs()), filters, fieldsQry.isEnforceJoinOrder(), fieldsQry.getTimeout(), cancel);
cur = new QueryCursorImpl<>(new Iterable<List<?>>() {
@@ -430,7 +408,7 @@ public class DmlStatementsProcessor {
*/
private UpdateResult processDmlSelectResult(GridCacheContext cctx, UpdatePlan plan, Iterable<List<?>> cursor,
int pageSize) throws IgniteCheckedException {
- switch (plan.mode) {
+ switch (plan.mode()) {
case MERGE:
return new UpdateResult(doMerge(plan, cursor, pageSize), X.EMPTY_OBJECT_ARRAY);
@@ -444,7 +422,7 @@ public class DmlStatementsProcessor {
return doDelete(cctx, cursor, pageSize);
default:
- throw new IgniteSQLException("Unexpected DML operation [mode=" + plan.mode + ']',
+ throw new IgniteSQLException("Unexpected DML operation [mode=" + plan.mode() + ']',
IgniteQueryErrorCode.UNEXPECTED_OPERATION);
}
}
@@ -480,46 +458,6 @@ public class DmlStatementsProcessor {
}
/**
- * Perform single cache operation based on given args.
- * @param plan Update plan.
- * @param args Query parameters.
- * @return 1 if an item was affected, 0 otherwise.
- * @throws IgniteCheckedException if failed.
- */
- @SuppressWarnings({"unchecked", "ConstantConditions"})
- private static UpdateResult doFastUpdate(UpdatePlan plan, Object[] args) throws IgniteCheckedException {
- GridCacheContext cctx = plan.tbl.rowDescriptor().context();
-
- FastUpdateArguments singleUpdate = plan.fastUpdateArgs;
-
- assert singleUpdate != null;
-
- boolean valBounded = (singleUpdate.val != FastUpdateArguments.NULL_ARGUMENT);
-
- if (singleUpdate.newVal != FastUpdateArguments.NULL_ARGUMENT) { // Single item UPDATE
- Object key = singleUpdate.key.apply(args);
- Object newVal = singleUpdate.newVal.apply(args);
-
- if (valBounded) {
- Object val = singleUpdate.val.apply(args);
-
- return (cctx.cache().replace(key, val, newVal) ? UpdateResult.ONE : UpdateResult.ZERO);
- }
- else
- return (cctx.cache().replace(key, newVal) ? UpdateResult.ONE : UpdateResult.ZERO);
- }
- else { // Single item DELETE
- Object key = singleUpdate.key.apply(args);
- Object val = singleUpdate.val.apply(args);
-
- if (singleUpdate.val == FastUpdateArguments.NULL_ARGUMENT) // No _val bound in source query
- return cctx.cache().remove(key) ? UpdateResult.ONE : UpdateResult.ZERO;
- else
- return cctx.cache().remove(key, val) ? UpdateResult.ONE : UpdateResult.ZERO;
- }
- }
-
- /**
* @param schemaName Schema name.
* @param fieldsQry Initial query.
* @param plan Update plan.
@@ -529,13 +467,15 @@ public class DmlStatementsProcessor {
*/
private UpdateResult doDistributedUpdate(String schemaName, SqlFieldsQuery fieldsQry, UpdatePlan plan,
GridQueryCancel cancel) throws IgniteCheckedException {
- assert plan.distributed != null;
+ DmlDistributedPlanInfo distributedPlan = plan.distributedPlan();
+
+ assert distributedPlan != null;
if (cancel == null)
cancel = new GridQueryCancel();
- return idx.runDistributedUpdate(schemaName, fieldsQry, plan.distributed.getCacheIds(),
- plan.distributed.isReplicatedOnly(), cancel);
+ return idx.runDistributedUpdate(schemaName, fieldsQry, distributedPlan.getCacheIds(),
+ distributedPlan.isReplicatedOnly(), cancel);
}
/**
@@ -548,7 +488,7 @@ public class DmlStatementsProcessor {
@SuppressWarnings({"unchecked", "ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
private UpdateResult doDelete(GridCacheContext cctx, Iterable<List<?>> cursor, int pageSize)
throws IgniteCheckedException {
- BatchSender sender = new BatchSender(cctx, pageSize);
+ DmlBatchSender sender = new DmlBatchSender(cctx, pageSize);
for (List<?> row : cursor) {
if (row.size() != 2) {
@@ -594,84 +534,18 @@ public class DmlStatementsProcessor {
@SuppressWarnings({"unchecked", "ThrowableResultOfMethodCallIgnored"})
private UpdateResult doUpdate(UpdatePlan plan, Iterable<List<?>> cursor, int pageSize)
throws IgniteCheckedException {
- GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
-
- GridCacheContext cctx = desc.context();
-
- boolean bin = cctx.binaryMarshaller();
-
- String[] updatedColNames = plan.colNames;
-
- int valColIdx = plan.valColIdx;
+ GridCacheContext cctx = plan.cacheContext();
- boolean hasNewVal = (valColIdx != -1);
-
- // Statement updates distinct properties if it does not have _val in updated columns list
- // or if its list of updated columns includes only _val, i.e. is single element.
- boolean hasProps = !hasNewVal || updatedColNames.length > 1;
-
- BatchSender sender = new BatchSender(cctx, pageSize);
+ DmlBatchSender sender = new DmlBatchSender(cctx, pageSize);
for (List<?> row : cursor) {
- Object key = row.get(0);
-
- Object newVal;
-
- Map<String, Object> newColVals = new HashMap<>();
-
- for (int i = 0; i < plan.colNames.length; i++) {
- if (hasNewVal && i == valColIdx - 2)
- continue;
-
- GridQueryProperty prop = plan.tbl.rowDescriptor().type().property(plan.colNames[i]);
-
- assert prop != null : "Unknown property: " + plan.colNames[i];
-
- newColVals.put(plan.colNames[i], convert(row.get(i + 2), desc, prop.type(), plan.colTypes[i]));
- }
-
- newVal = plan.valSupplier.apply(row);
-
- if (newVal == null)
- throw new IgniteSQLException("New value for UPDATE must not be null", IgniteQueryErrorCode.NULL_VALUE);
-
- // Skip key and value - that's why we start off with 3rd column
- for (int i = 0; i < plan.tbl.getColumns().length - DEFAULT_COLUMNS_COUNT; i++) {
- Column c = plan.tbl.getColumn(i + DEFAULT_COLUMNS_COUNT);
-
- if (desc.isKeyValueOrVersionColumn(c.getColumnId()))
- continue;
-
- GridQueryProperty prop = desc.type().property(c.getName());
-
- if (prop.key())
- continue; // Don't get values of key's columns - we won't use them anyway
-
- boolean hasNewColVal = newColVals.containsKey(c.getName());
-
- if (!hasNewColVal)
- continue;
-
- Object colVal = newColVals.get(c.getName());
-
- // UPDATE currently does not allow to modify key or its fields, so we must be safe to pass null as key.
- desc.setColumnValue(null, newVal, colVal, i);
- }
-
- if (bin && hasProps) {
- assert newVal instanceof BinaryObjectBuilder;
-
- newVal = ((BinaryObjectBuilder) newVal).build();
- }
-
- desc.type().validateKeyAndValue(key, newVal);
-
- Object srcVal = row.get(1);
+ T3<Object, Object, Object> row0 = plan.processRowForUpdate(row);
- if (bin && !(srcVal instanceof BinaryObject))
- srcVal = cctx.grid().binary().toBinary(srcVal);
+ Object key = row0.get1();
+ Object oldVal = row0.get2();
+ Object newVal = row0.get3();
- sender.add(key, new ModifyingEntryProcessor(srcVal, new EntryValueUpdater(newVal)));
+ sender.add(key, new ModifyingEntryProcessor(oldVal, new EntryValueUpdater(newVal)));
}
sender.flush();
@@ -699,120 +573,6 @@ public class DmlStatementsProcessor {
}
/**
- * Convert value to column's expected type by means of H2.
- *
- * @param val Source value.
- * @param desc Row descriptor.
- * @param expCls Expected value class.
- * @param type Expected column type to convert to.
- * @return Converted object.
- * @throws IgniteCheckedException if failed.
- */
- @SuppressWarnings({"ConstantConditions", "SuspiciousSystemArraycopy"})
- private static Object convert(Object val, GridH2RowDescriptor desc, Class<?> expCls, int type)
- throws IgniteCheckedException {
- if (val == null)
- return null;
-
- Class<?> currCls = val.getClass();
-
- try {
- if (val instanceof Date && currCls != Date.class && expCls == Date.class) {
- // H2 thinks that java.util.Date is always a Timestamp, while binary marshaller expects
- // precise Date instance. Let's satisfy it.
- return new Date(((Date) val).getTime());
- }
-
- // User-given UUID is always serialized by H2 to byte array, so we have to deserialize manually
- if (type == Value.UUID && currCls == byte[].class)
- return U.unmarshal(desc.context().marshaller(), (byte[]) val,
- U.resolveClassLoader(desc.context().gridConfig()));
-
- if (LocalDateTimeUtils.isJava8DateApiPresent()) {
- if (val instanceof Timestamp && LocalDateTimeUtils.isLocalDateTime(expCls))
- return LocalDateTimeUtils.valueToLocalDateTime(ValueTimestamp.get((Timestamp) val));
-
- if (val instanceof Date && LocalDateTimeUtils.isLocalDate(expCls))
- return LocalDateTimeUtils.valueToLocalDate(ValueDate.fromDateValue(
- DateTimeUtils.dateValueFromDate(((Date) val).getTime())));
-
- if (val instanceof Time && LocalDateTimeUtils.isLocalTime(expCls))
- return LocalDateTimeUtils.valueToLocalTime(ValueTime.get((Time) val));
- }
-
- // We have to convert arrays of reference types manually -
- // see https://issues.apache.org/jira/browse/IGNITE-4327
- // Still, we only can convert from Object[] to something more precise.
- if (type == Value.ARRAY && currCls != expCls) {
- if (currCls != Object[].class)
- throw new IgniteCheckedException("Unexpected array type - only conversion from Object[] " +
- "is assumed");
-
- // Why would otherwise type be Value.ARRAY?
- assert expCls.isArray();
-
- Object[] curr = (Object[]) val;
-
- Object newArr = Array.newInstance(expCls.getComponentType(), curr.length);
-
- System.arraycopy(curr, 0, newArr, 0, curr.length);
-
- return newArr;
- }
-
- return H2Utils.convert(val, desc, type);
- }
- catch (Exception e) {
- throw new IgniteSQLException("Value conversion failed [from=" + currCls.getName() + ", to=" +
- expCls.getName() +']', IgniteQueryErrorCode.CONVERSION_FAILED, e);
- }
- }
-
- /**
- * Process errors of entry processor - split the keys into duplicated/concurrently modified and those whose
- * processing yielded an exception.
- *
- * @param res Result of {@link GridCacheAdapter#invokeAll)}
- * @return pair [array of duplicated/concurrently modified keys, SQL exception for erroneous keys] (exception is
- * null if all keys are duplicates/concurrently modified ones).
- */
- private static PageProcessingErrorResult splitErrors(Map<Object, EntryProcessorResult<Boolean>> res) {
- Set<Object> errKeys = new LinkedHashSet<>(res.keySet());
-
- SQLException currSqlEx = null;
-
- SQLException firstSqlEx = null;
-
- int errors = 0;
-
- // Let's form a chain of SQL exceptions
- for (Map.Entry<Object, EntryProcessorResult<Boolean>> e : res.entrySet()) {
- try {
- e.getValue().get();
- }
- catch (EntryProcessorException ex) {
- SQLException next = createJdbcSqlException("Failed to process key '" + e.getKey() + '\'',
- IgniteQueryErrorCode.ENTRY_PROCESSING);
-
- next.initCause(ex);
-
- if (currSqlEx != null)
- currSqlEx.setNextException(next);
- else
- firstSqlEx = next;
-
- currSqlEx = next;
-
- errKeys.remove(e.getKey());
-
- errors++;
- }
- }
-
- return new PageProcessingErrorResult(errKeys.toArray(), firstSqlEx, errors);
- }
-
- /**
* Execute MERGE statement plan.
* @param cursor Cursor to take inserted data from.
* @param pageSize Batch size to stream data from {@code cursor}, anything <= 0 for single page operations.
@@ -821,13 +581,11 @@ public class DmlStatementsProcessor {
*/
@SuppressWarnings("unchecked")
private long doMerge(UpdatePlan plan, Iterable<List<?>> cursor, int pageSize) throws IgniteCheckedException {
- GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
-
- GridCacheContext cctx = desc.context();
+ GridCacheContext cctx = plan.cacheContext();
// If we have just one item to put, just do so
- if (plan.rowsNum == 1) {
- IgniteBiTuple t = rowToKeyValue(cctx, cursor.iterator().next(), plan);
+ if (plan.rowCount() == 1) {
+ IgniteBiTuple t = plan.processRow(cursor.iterator().next());
cctx.cache().put(t.getKey(), t.getValue());
@@ -841,7 +599,7 @@ public class DmlStatementsProcessor {
for (Iterator<List<?>> it = cursor.iterator(); it.hasNext();) {
List<?> row = it.next();
- IgniteBiTuple t = rowToKeyValue(cctx, row, plan);
+ IgniteBiTuple t = plan.processRow(row);
rows.put(t.getKey(), t.getValue());
@@ -868,13 +626,11 @@ public class DmlStatementsProcessor {
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
private long doInsert(UpdatePlan plan, Iterable<List<?>> cursor, int pageSize) throws IgniteCheckedException {
- GridH2RowDescriptor desc = plan.tbl.rowDescriptor();
-
- GridCacheContext cctx = desc.context();
+ GridCacheContext cctx = plan.cacheContext();
// If we have just one item to put, just do so
- if (plan.rowsNum == 1) {
- IgniteBiTuple t = rowToKeyValue(cctx, cursor.iterator().next(), plan);
+ if (plan.rowCount() == 1) {
+ IgniteBiTuple t = plan.processRow(cursor.iterator().next());
if (cctx.cache().putIfAbsent(t.getKey(), t.getValue()))
return 1;
@@ -884,10 +640,10 @@ public class DmlStatementsProcessor {
}
else {
// Keys that failed to INSERT due to duplication.
- BatchSender sender = new BatchSender(cctx, pageSize);
+ DmlBatchSender sender = new DmlBatchSender(cctx, pageSize);
for (List<?> row : cursor) {
- final IgniteBiTuple keyValPair = rowToKeyValue(cctx, row, plan);
+ final IgniteBiTuple keyValPair = plan.processRow(row);
sender.add(keyValPair.getKey(), new InsertEntryProcessor(keyValPair.getValue()));
}
@@ -916,124 +672,6 @@ public class DmlStatementsProcessor {
}
/**
- * Execute given entry processors and collect errors, if any.
- * @param cctx Cache context.
- * @param rows Rows to process.
- * @return Triple [number of rows actually changed; keys that failed to update (duplicates or concurrently
- * updated ones); chain of exceptions for all keys whose processing resulted in error, or null for no errors].
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings({"unchecked", "ConstantConditions"})
- private static PageProcessingResult processPage(GridCacheContext cctx,
- Map<Object, EntryProcessor<Object, Object, Boolean>> rows) throws IgniteCheckedException {
- Map<Object, EntryProcessorResult<Boolean>> res = cctx.cache().invokeAll(rows);
-
- if (F.isEmpty(res))
- return new PageProcessingResult(rows.size(), null, null);
-
- PageProcessingErrorResult splitRes = splitErrors(res);
-
- int keysCnt = splitRes.errKeys.length;
-
- return new PageProcessingResult(rows.size() - keysCnt - splitRes.cnt, splitRes.errKeys, splitRes.ex);
- }
-
- /**
- * Convert row presented as an array of Objects into key-value pair to be inserted to cache.
- * @param cctx Cache context.
- * @param row Row to process.
- * @param plan Update plan.
- * @throws IgniteCheckedException if failed.
- */
- @SuppressWarnings({"unchecked", "ConstantConditions", "ResultOfMethodCallIgnored"})
- private IgniteBiTuple<?, ?> rowToKeyValue(GridCacheContext cctx, List<?> row, UpdatePlan plan)
- throws IgniteCheckedException {
- GridH2RowDescriptor rowDesc = plan.tbl.rowDescriptor();
- GridQueryTypeDescriptor desc = rowDesc.type();
-
- Object key = plan.keySupplier.apply(row);
-
- if (QueryUtils.isSqlType(desc.keyClass())) {
- assert plan.keyColIdx != -1;
-
- key = convert(key, rowDesc, desc.keyClass(), plan.colTypes[plan.keyColIdx]);
- }
-
- Object val = plan.valSupplier.apply(row);
-
- if (QueryUtils.isSqlType(desc.valueClass())) {
- assert plan.valColIdx != -1;
-
- val = convert(val, rowDesc, desc.valueClass(), plan.colTypes[plan.valColIdx]);
- }
-
- if (key == null) {
- if (F.isEmpty(desc.keyFieldName()))
- throw new IgniteSQLException("Key for INSERT or MERGE must not be null", IgniteQueryErrorCode.NULL_KEY);
- else
- throw new IgniteSQLException("Null value is not allowed for column '" + desc.keyFieldName() + "'",
- IgniteQueryErrorCode.NULL_KEY);
- }
-
- if (val == null) {
- if (F.isEmpty(desc.valueFieldName()))
- throw new IgniteSQLException("Value for INSERT, MERGE, or UPDATE must not be null",
- IgniteQueryErrorCode.NULL_VALUE);
- else
- throw new IgniteSQLException("Null value is not allowed for column '" + desc.valueFieldName() + "'",
- IgniteQueryErrorCode.NULL_VALUE);
- }
-
- Map<String, Object> newColVals = new HashMap<>();
-
- for (int i = 0; i < plan.colNames.length; i++) {
- if (i == plan.keyColIdx || i == plan.valColIdx)
- continue;
-
- String colName = plan.colNames[i];
-
- GridQueryProperty prop = desc.property(colName);
-
- assert prop != null;
-
- Class<?> expCls = prop.type();
-
- newColVals.put(colName, convert(row.get(i), rowDesc, expCls, plan.colTypes[i]));
- }
-
- // We update columns in the order specified by the table for a reason - table's
- // column order preserves their precedence for correct update of nested properties.
- Column[] cols = plan.tbl.getColumns();
-
- // First 3 columns are _key, _val and _ver. Skip 'em.
- for (int i = DEFAULT_COLUMNS_COUNT; i < cols.length; i++) {
- if (plan.tbl.rowDescriptor().isKeyValueOrVersionColumn(i))
- continue;
-
- String colName = cols[i].getName();
-
- if (!newColVals.containsKey(colName))
- continue;
-
- Object colVal = newColVals.get(colName);
-
- desc.setValue(colName, key, val, colVal);
- }
-
- if (cctx.binaryMarshaller()) {
- if (key instanceof BinaryObjectBuilder)
- key = ((BinaryObjectBuilder) key).build();
-
- if (val instanceof BinaryObjectBuilder)
- val = ((BinaryObjectBuilder) val).build();
- }
-
- desc.validateKeyAndValue(key, val);
-
- return new IgniteBiTuple<>(key, val);
- }
-
- /**
*
* @param schemaName Schema name.
* @param stmt Prepared statement.
@@ -1164,7 +802,7 @@ public class DmlStatementsProcessor {
static void checkUpdateResult(UpdateResult r) {
if (!F.isEmpty(r.errorKeys())) {
String msg = "Failed to update some keys because they had been modified concurrently " +
- "[keys=" + r.errorKeys() + ']';
+ "[keys=" + Arrays.toString(r.errorKeys()) + ']';
SQLException conEx = createJdbcSqlException(msg, IgniteQueryErrorCode.CONCURRENT_UPDATE);
@@ -1172,175 +810,4 @@ public class DmlStatementsProcessor {
}
}
- /** Result of processing an individual page with {@link IgniteCache#invokeAll} including error details, if any. */
- private final static class PageProcessingResult {
- /** Number of successfully processed items. */
- final long cnt;
-
- /** Keys that failed to be updated or deleted due to concurrent modification of values. */
- @NotNull
- final Object[] errKeys;
-
- /** Chain of exceptions corresponding to failed keys. Null if no keys yielded an exception. */
- final SQLException ex;
-
- /** */
- @SuppressWarnings("ConstantConditions")
- private PageProcessingResult(long cnt, Object[] errKeys, SQLException ex) {
- this.cnt = cnt;
- this.errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY);
- this.ex = ex;
- }
- }
-
- /** Result of splitting keys whose processing resulted into an exception from those skipped by
- * logic of {@link EntryProcessor}s (most likely INSERT duplicates, or UPDATE/DELETE keys whose values
- * had been modified concurrently), counting and collecting entry processor exceptions.
- */
- private final static class PageProcessingErrorResult {
- /** Keys that failed to be processed by {@link EntryProcessor} (not due to an exception). */
- @NotNull
- final Object[] errKeys;
-
- /** Number of entries whose processing resulted into an exception. */
- final int cnt;
-
- /** Chain of exceptions corresponding to failed keys. Null if no keys yielded an exception. */
- final SQLException ex;
-
- /** */
- @SuppressWarnings("ConstantConditions")
- private PageProcessingErrorResult(@NotNull Object[] errKeys, SQLException ex, int exCnt) {
- errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY);
- // When exceptions count must be zero, exceptions chain must be not null, and vice versa.
- assert exCnt == 0 ^ ex != null;
-
- this.errKeys = errKeys;
- this.cnt = exCnt;
- this.ex = ex;
- }
- }
-
- /**
- * Batch sender class.
- */
- private static class BatchSender {
- /** Cache context. */
- private final GridCacheContext cctx;
-
- /** Batch size. */
- private final int size;
-
- /** Batches. */
- private final Map<UUID, Map<Object, EntryProcessor<Object, Object, Boolean>>> batches = new HashMap<>();
-
- /** Result count. */
- private long updateCnt;
-
- /** Failed keys. */
- private List<Object> failedKeys;
-
- /** Exception. */
- private SQLException err;
-
- /**
- * Constructor.
- *
- * @param cctx Cache context.
- * @param size Batch.
- */
- public BatchSender(GridCacheContext cctx, int size) {
- this.cctx = cctx;
- this.size = size;
- }
-
- /**
- * Add entry to batch.
- *
- * @param key Key.
- * @param proc Processor.
- */
- public void add(Object key, EntryProcessor<Object, Object, Boolean> proc) throws IgniteCheckedException {
- ClusterNode node = cctx.affinity().primaryByKey(key, AffinityTopologyVersion.NONE);
-
- if (node == null)
- throw new IgniteCheckedException("Failed to map key to node.");
-
- UUID nodeId = node.id();
-
- Map<Object, EntryProcessor<Object, Object, Boolean>> batch = batches.get(nodeId);
-
- if (batch == null) {
- batch = new HashMap<>();
-
- batches.put(nodeId, batch);
- }
-
- batch.put(key, proc);
-
- if (batch.size() >= size) {
- sendBatch(batch);
-
- batch.clear();
- }
- }
-
- /**
- * Flush any remaining entries.
- *
- * @throws IgniteCheckedException If failed.
- */
- public void flush() throws IgniteCheckedException {
- for (Map<Object, EntryProcessor<Object, Object, Boolean>> batch : batches.values()) {
- if (!batch.isEmpty())
- sendBatch(batch);
- }
- }
-
- /**
- * @return Update count.
- */
- public long updateCount() {
- return updateCnt;
- }
-
- /**
- * @return Failed keys.
- */
- public List<Object> failedKeys() {
- return failedKeys != null ? failedKeys : Collections.emptyList();
- }
-
- /**
- * @return Error.
- */
- public SQLException error() {
- return err;
- }
-
- /**
- * Send the batch.
- *
- * @param batch Batch.
- * @throws IgniteCheckedException If failed.
- */
- private void sendBatch(Map<Object, EntryProcessor<Object, Object, Boolean>> batch)
- throws IgniteCheckedException {
- PageProcessingResult pageRes = processPage(cctx, batch);
-
- updateCnt += pageRes.cnt;
-
- if (failedKeys == null)
- failedKeys = new ArrayList<>();
-
- failedKeys.addAll(F.asList(pageRes.errKeys));
-
- if (pageRes.ex != null) {
- if (err == null)
- err = pageRes.ex;
- else
- err.setNextException(pageRes.ex);
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java
index de0e63f..32381ba 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/UpdateResult.java
@@ -25,10 +25,10 @@ import org.apache.ignite.internal.util.typedef.internal.U;
*/
public final class UpdateResult {
/** Result to return for operations that affected 1 item - mostly to be used for fast updates and deletes. */
- final static UpdateResult ONE = new UpdateResult(1, X.EMPTY_OBJECT_ARRAY);
+ public static final UpdateResult ONE = new UpdateResult(1, X.EMPTY_OBJECT_ARRAY);
/** Result to return for operations that affected 0 items - mostly to be used for fast updates and deletes. */
- final static UpdateResult ZERO = new UpdateResult(0, X.EMPTY_OBJECT_ARRAY);
+ public static final UpdateResult ZERO = new UpdateResult(0, X.EMPTY_OBJECT_ARRAY);
/** Number of processed items. */
private final long cnt;
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
new file mode 100644
index 0000000..054e708
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
@@ -0,0 +1,609 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAlias;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlArray;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlAst;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlColumn;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlDelete;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunction;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlFunctionType;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlJoin;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlKeyword;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperation;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlOperationType;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuery;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSelect;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlSubquery;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlTable;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlType;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUnion;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlUpdate;
+import org.apache.ignite.internal.util.lang.IgnitePair;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.h2.command.Parser;
+import org.h2.expression.Expression;
+import org.h2.table.Column;
+import org.h2.table.Table;
+import org.h2.util.IntArray;
+import org.h2.value.DataType;
+import org.h2.value.Value;
+import org.h2.value.ValueDate;
+import org.h2.value.ValueInt;
+import org.h2.value.ValueString;
+import org.h2.value.ValueTime;
+import org.h2.value.ValueTimestamp;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * AST utils for DML
+ */
+public final class DmlAstUtils {
+ /**
+ * Empty ctor to prevent initialization.
+ */
+ private DmlAstUtils() {
+ // No-op.
+ }
+
+ /**
+ * Create SELECT on which subsequent INSERT or MERGE will be based.
+ *
+ * @param cols Columns to insert values into.
+ * @param rows Rows to create pseudo-SELECT upon.
+ * @param subQry Subquery to use rather than rows.
+ * @return Subquery or pseudo-SELECT to evaluate inserted expressions.
+ */
+ public static GridSqlQuery selectForInsertOrMerge(GridSqlColumn[] cols, List<GridSqlElement[]> rows,
+ GridSqlQuery subQry) {
+ if (!F.isEmpty(rows)) {
+ assert !F.isEmpty(cols);
+
+ GridSqlSelect sel = new GridSqlSelect();
+
+ GridSqlFunction from = new GridSqlFunction(GridSqlFunctionType.TABLE);
+
+ sel.from(from);
+
+ GridSqlArray[] args = new GridSqlArray[cols.length];
+
+ for (int i = 0; i < cols.length; i++) {
+ GridSqlArray arr = new GridSqlArray(rows.size());
+
+ String colName = cols[i].columnName();
+
+ GridSqlAlias alias = new GridSqlAlias(colName, arr);
+
+ alias.resultType(cols[i].resultType());
+
+ from.addChild(alias);
+
+ args[i] = arr;
+
+ GridSqlColumn newCol = new GridSqlColumn(null, from, null,"TABLE", colName);
+
+ newCol.resultType(cols[i].resultType());
+
+ sel.addColumn(newCol, true);
+ }
+
+ for (GridSqlElement[] row : rows) {
+ assert cols.length == row.length;
+
+ for (int i = 0; i < row.length; i++)
+ args[i].addChild(row[i]);
+ }
+
+ return sel;
+ }
+ else {
+ assert subQry != null;
+
+ return subQry;
+ }
+ }
+
+ /**
+ * Generate SQL SELECT based on DELETE's WHERE, LIMIT, etc.
+ *
+ * @param del Delete statement.
+ * @param keysParamIdx Index for .
+ * @return SELECT statement.
+ */
+ public static GridSqlSelect selectForDelete(GridSqlDelete del, @Nullable Integer keysParamIdx) {
+ GridSqlSelect mapQry = new GridSqlSelect();
+
+ mapQry.from(del.from());
+
+ Set<GridSqlTable> tbls = new HashSet<>();
+
+ collectAllGridTablesInTarget(del.from(), tbls);
+
+ assert tbls.size() == 1 : "Failed to determine target table for DELETE";
+
+ GridSqlTable tbl = tbls.iterator().next();
+
+ GridH2Table gridTbl = tbl.dataTable();
+
+ assert gridTbl != null : "Failed to determine target grid table for DELETE";
+
+ Column h2KeyCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.KEY_COL);
+
+ Column h2ValCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.VAL_COL);
+
+ GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName());
+ keyCol.resultType(GridSqlType.fromColumn(h2KeyCol));
+
+ GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName());
+ valCol.resultType(GridSqlType.fromColumn(h2ValCol));
+
+ mapQry.addColumn(keyCol, true);
+ mapQry.addColumn(valCol, true);
+
+ GridSqlElement where = del.where();
+ if (keysParamIdx != null)
+ where = injectKeysFilterParam(where, keyCol, keysParamIdx);
+
+ mapQry.where(where);
+ mapQry.limit(del.limit());
+
+ return mapQry;
+ }
+
+ /**
+ * @param update UPDATE statement.
+ * @return {@code null} if given statement directly updates {@code _val} column with a literal or param value
+ * and filters by single non expression key (and, optionally, by single non expression value).
+ */
+ public static FastUpdate getFastUpdateArgs(GridSqlUpdate update) {
+ IgnitePair<GridSqlElement> filter = findKeyValueEqualityCondition(update.where());
+
+ if (filter == null)
+ return null;
+
+ if (update.cols().size() != 1)
+ return null;
+
+ Table tbl = update.cols().get(0).column().getTable();
+ if (!(tbl instanceof GridH2Table))
+ return null;
+
+ GridH2RowDescriptor desc = ((GridH2Table)tbl).rowDescriptor();
+ if (!desc.isValueColumn(update.cols().get(0).column().getColumnId()))
+ return null;
+
+ GridSqlElement set = update.set().get(update.cols().get(0).columnName());
+
+ if (!(set instanceof GridSqlConst || set instanceof GridSqlParameter))
+ return null;
+
+ return FastUpdate.create(filter.getKey(), filter.getValue(), set);
+ }
+
+ /**
+ * @param del DELETE statement.
+ * @return {@code true} if given statement filters by single non expression key.
+ */
+ public static FastUpdate getFastDeleteArgs(GridSqlDelete del) {
+ IgnitePair<GridSqlElement> filter = findKeyValueEqualityCondition(del.where());
+
+ if (filter == null)
+ return null;
+
+ return FastUpdate.create(filter.getKey(), filter.getValue(), null);
+ }
+
+ /**
+ * @param where Element to test.
+ * @return Whether given element corresponds to {@code WHERE _key = ?}, and key is a literal expressed
+ * in query or a query param.
+ */
+ @SuppressWarnings("RedundantCast")
+ private static IgnitePair<GridSqlElement> findKeyValueEqualityCondition(GridSqlElement where) {
+ if (where == null || !(where instanceof GridSqlOperation))
+ return null;
+
+ GridSqlOperation whereOp = (GridSqlOperation) where;
+
+ // Does this WHERE limit only by _key?
+ if (isKeyEqualityCondition(whereOp))
+ return new IgnitePair<>((GridSqlElement)whereOp.child(1), null);
+
+ // Or maybe it limits both by _key and _val?
+ if (whereOp.operationType() != GridSqlOperationType.AND)
+ return null;
+
+ GridSqlElement left = whereOp.child(0);
+
+ GridSqlElement right = whereOp.child(1);
+
+ if (!(left instanceof GridSqlOperation && right instanceof GridSqlOperation))
+ return null;
+
+ GridSqlOperation leftOp = (GridSqlOperation) left;
+
+ GridSqlOperation rightOp = (GridSqlOperation) right;
+
+ if (isKeyEqualityCondition(leftOp)) { // _key = ? and _val = ?
+ if (!isValueEqualityCondition(rightOp))
+ return null;
+
+ return new IgnitePair<>((GridSqlElement)leftOp.child(1), (GridSqlElement)rightOp.child(1));
+ }
+ else if (isKeyEqualityCondition(rightOp)) { // _val = ? and _key = ?
+ if (!isValueEqualityCondition(leftOp))
+ return null;
+
+ return new IgnitePair<>((GridSqlElement)rightOp.child(1), (GridSqlElement)leftOp.child(1));
+ }
+ else // Neither
+ return null;
+ }
+
+ /**
+ * @param op Operation.
+ * @param key true - check for key equality condition,
+ * otherwise check for value equality condition
+ * @return Whether this condition is of form {@code colName} = ?
+ */
+ private static boolean isEqualityCondition(GridSqlOperation op, boolean key) {
+ if (op.operationType() != GridSqlOperationType.EQUAL)
+ return false;
+
+ GridSqlElement left = op.child(0);
+ GridSqlElement right = op.child(1);
+
+ if (!(left instanceof GridSqlColumn))
+ return false;
+
+ GridSqlColumn column = (GridSqlColumn)left;
+ if (!(column.column().getTable() instanceof GridH2Table))
+ return false;
+
+ GridH2RowDescriptor desc =((GridH2Table) column.column().getTable()).rowDescriptor();
+
+ return (key ? desc.isKeyColumn(column.column().getColumnId()) :
+ desc.isValueColumn(column.column().getColumnId())) &&
+ (right instanceof GridSqlConst || right instanceof GridSqlParameter);
+ }
+
+ /**
+ * @param op Operation.
+ * @return Whether this condition is of form _key = ?
+ */
+ private static boolean isKeyEqualityCondition(GridSqlOperation op) {
+ return isEqualityCondition(op, true);
+ }
+
+ /**
+ * @param op Operation.
+ * @return Whether this condition is of form _val = ?
+ */
+ private static boolean isValueEqualityCondition(GridSqlOperation op) {
+ return isEqualityCondition(op, false);
+ }
+
+
+ /**
+ * Generate SQL SELECT based on UPDATE's WHERE, LIMIT, etc.
+ *
+ * @param update Update statement.
+ * @param keysParamIdx Index of new param for the array of keys.
+ * @return SELECT statement.
+ */
+ public static GridSqlSelect selectForUpdate(GridSqlUpdate update, @Nullable Integer keysParamIdx) {
+ GridSqlSelect mapQry = new GridSqlSelect();
+
+ mapQry.from(update.target());
+
+ Set<GridSqlTable> tbls = new HashSet<>();
+
+ collectAllGridTablesInTarget(update.target(), tbls);
+
+ assert tbls.size() == 1 : "Failed to determine target table for UPDATE";
+
+ GridSqlTable tbl = tbls.iterator().next();
+
+ GridH2Table gridTbl = tbl.dataTable();
+
+ assert gridTbl != null : "Failed to determine target grid table for UPDATE";
+
+ Column h2KeyCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.KEY_COL);
+
+ Column h2ValCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.VAL_COL);
+
+ GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName());
+ keyCol.resultType(GridSqlType.fromColumn(h2KeyCol));
+
+ GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName());
+ valCol.resultType(GridSqlType.fromColumn(h2ValCol));
+
+ mapQry.addColumn(keyCol, true);
+ mapQry.addColumn(valCol, true);
+
+ for (GridSqlColumn c : update.cols()) {
+ String newColName = Parser.quoteIdentifier("_upd_" + c.columnName());
+ // We have to use aliases to cover cases when the user
+ // wants to update _val field directly (if it's a literal)
+ GridSqlAlias alias = new GridSqlAlias(newColName, elementOrDefault(update.set().get(c.columnName()), c), true);
+ alias.resultType(c.resultType());
+ mapQry.addColumn(alias, true);
+ }
+
+ GridSqlElement where = update.where();
+ if (keysParamIdx != null)
+ where = injectKeysFilterParam(where, keyCol, keysParamIdx);
+
+ mapQry.where(where);
+ mapQry.limit(update.limit());
+
+ return mapQry;
+ }
+
+ /**
+ * Do what we can to compute default value for this column (mimics H2 behavior).
+ * @see Table#getDefaultValue
+ * @see Column#validateConvertUpdateSequence
+ * @param el SQL element.
+ * @param col Column.
+ * @return {@link GridSqlConst#NULL}, if {@code el} is null, or {@code el} if
+ * it's not {@link GridSqlKeyword#DEFAULT}, or computed default value.
+ */
+ private static GridSqlElement elementOrDefault(GridSqlElement el, GridSqlColumn col) {
+ if (el == null)
+ return GridSqlConst.NULL;
+
+ if (el != GridSqlKeyword.DEFAULT)
+ return el;
+
+ Column h2Col = col.column();
+
+ Expression dfltExpr = h2Col.getDefaultExpression();
+
+ Value dfltVal;
+
+ try {
+ dfltVal = dfltExpr != null ? dfltExpr.getValue(null) : null;
+ }
+ catch (Exception ignored) {
+ throw new IgniteSQLException("Failed to evaluate default value for a column " + col.columnName());
+ }
+
+ if (dfltVal != null)
+ return new GridSqlConst(dfltVal);
+
+ int type = h2Col.getType();
+
+ DataType dt = DataType.getDataType(type);
+
+ if (dt.decimal)
+ dfltVal = ValueInt.get(0).convertTo(type);
+ else if (dt.type == Value.TIMESTAMP)
+ dfltVal = ValueTimestamp.fromMillis(U.currentTimeMillis());
+ else if (dt.type == Value.TIME)
+ dfltVal = ValueTime.fromNanos(0);
+ else if (dt.type == Value.DATE)
+ dfltVal = ValueDate.fromMillis(U.currentTimeMillis());
+ else
+ dfltVal = ValueString.get("").convertTo(type);
+
+ return new GridSqlConst(dfltVal);
+ }
+
+ /**
+ * Append additional condition to WHERE for it to select only specific keys.
+ *
+ * @param where Initial condition.
+ * @param keyCol Column to base the new condition on.
+ * @return New condition.
+ */
+ private static GridSqlElement injectKeysFilterParam(GridSqlElement where, GridSqlColumn keyCol, int paramIdx) {
+ // Yes, we need a subquery for "WHERE _key IN ?" to work with param being an array without dirty query rewriting.
+ GridSqlSelect sel = new GridSqlSelect();
+
+ GridSqlFunction from = new GridSqlFunction(GridSqlFunctionType.TABLE);
+
+ sel.from(from);
+
+ GridSqlColumn col = new GridSqlColumn(null, from, null, "TABLE", "_IGNITE_ERR_KEYS");
+
+ sel.addColumn(col, true);
+
+ GridSqlAlias alias = new GridSqlAlias("_IGNITE_ERR_KEYS", new GridSqlParameter(paramIdx));
+
+ alias.resultType(keyCol.resultType());
+
+ from.addChild(alias);
+
+ GridSqlElement e = new GridSqlOperation(GridSqlOperationType.IN, keyCol, new GridSqlSubquery(sel));
+
+ if (where == null)
+ return e;
+ else
+ return new GridSqlOperation(GridSqlOperationType.AND, where, e);
+ }
+
+ /**
+ * @param qry Select.
+ * @param params Parameters.
+ * @param target Extracted parameters.
+ * @param paramIdxs Parameter indexes.
+ * @return Extracted parameters list.
+ */
+ @SuppressWarnings("unused")
+ private static List<Object> findParams(GridSqlQuery qry, Object[] params, ArrayList<Object> target,
+ IntArray paramIdxs) {
+ if (qry instanceof GridSqlSelect)
+ return findParams((GridSqlSelect)qry, params, target, paramIdxs);
+
+ GridSqlUnion union = (GridSqlUnion)qry;
+
+ findParams(union.left(), params, target, paramIdxs);
+ findParams(union.right(), params, target, paramIdxs);
+
+ findParams((GridSqlElement)qry.limit(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.offset(), params, target, paramIdxs);
+
+ return target;
+ }
+
+ /**
+ * @param qry Select.
+ * @param params Parameters.
+ * @param target Extracted parameters.
+ * @param paramIdxs Parameter indexes.
+ * @return Extracted parameters list.
+ */
+ private static List<Object> findParams(GridSqlSelect qry, Object[] params, ArrayList<Object> target,
+ IntArray paramIdxs) {
+ if (params.length == 0)
+ return target;
+
+ for (GridSqlAst el : qry.columns(false))
+ findParams((GridSqlElement)el, params, target, paramIdxs);
+
+ findParams((GridSqlElement)qry.from(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.where(), params, target, paramIdxs);
+
+ // Don't search in GROUP BY and HAVING since they expected to be in select list.
+
+ findParams((GridSqlElement)qry.limit(), params, target, paramIdxs);
+ findParams((GridSqlElement)qry.offset(), params, target, paramIdxs);
+
+ return target;
+ }
+
+ /**
+ * @param el Element.
+ * @param params Parameters.
+ * @param target Extracted parameters.
+ * @param paramIdxs Parameter indexes.
+ */
+ private static void findParams(@Nullable GridSqlElement el, Object[] params, ArrayList<Object> target,
+ IntArray paramIdxs) {
+ if (el == null)
+ return;
+
+ if (el instanceof GridSqlParameter) {
+ // H2 Supports queries like "select ?5" but first 4 non-existing parameters are need to be set to any value.
+ // Here we will set them to NULL.
+ final int idx = ((GridSqlParameter)el).index();
+
+ while (target.size() < idx)
+ target.add(null);
+
+ if (params.length <= idx)
+ throw new IgniteException("Invalid number of query parameters. " +
+ "Cannot find " + idx + " parameter.");
+
+ Object param = params[idx];
+
+ if (idx == target.size())
+ target.add(param);
+ else
+ target.set(idx, param);
+
+ paramIdxs.add(idx);
+ }
+ else if (el instanceof GridSqlSubquery)
+ findParams(((GridSqlSubquery)el).subquery(), params, target, paramIdxs);
+ else
+ for (int i = 0; i < el.size(); i++)
+ findParams((GridSqlElement)el.child(i), params, target, paramIdxs);
+ }
+
+ /**
+ * Processes all the tables and subqueries using the given closure.
+ *
+ * @param from FROM element.
+ * @param c Closure each found table and subquery will be passed to. If returns {@code true} the we need to stop.
+ * @return {@code true} If we have found.
+ */
+ @SuppressWarnings("RedundantCast")
+ private static boolean findTablesInFrom(GridSqlElement from, IgnitePredicate<GridSqlElement> c) {
+ if (from == null)
+ return false;
+
+ if (from instanceof GridSqlTable || from instanceof GridSqlSubquery)
+ return c.apply(from);
+
+ if (from instanceof GridSqlJoin) {
+ // Left and right.
+ if (findTablesInFrom((GridSqlElement)from.child(0), c))
+ return true;
+
+ if (findTablesInFrom((GridSqlElement)from.child(1), c))
+ return true;
+
+ // We don't process ON condition because it is not a joining part of from here.
+ return false;
+ }
+ else if (from instanceof GridSqlAlias)
+ return findTablesInFrom((GridSqlElement)from.child(), c);
+ else if (from instanceof GridSqlFunction)
+ return false;
+
+ throw new IllegalStateException(from.getClass().getName() + " : " + from.getSQL());
+ }
+
+ /**
+ * @param from From element.
+ * @param tbls Tables.
+ */
+ public static void collectAllGridTablesInTarget(GridSqlElement from, final Set<GridSqlTable> tbls) {
+ findTablesInFrom(from, new IgnitePredicate<GridSqlElement>() {
+ @Override public boolean apply(GridSqlElement el) {
+ if (el instanceof GridSqlTable)
+ tbls.add((GridSqlTable)el);
+
+ return false;
+ }
+ });
+ }
+
+ /**
+ * @param target Expression to extract the table from.
+ * @return Back end table for this element.
+ */
+ public static GridSqlTable gridTableForElement(GridSqlElement target) {
+ Set<GridSqlTable> tbls = new HashSet<>();
+
+ collectAllGridTablesInTarget(target, tbls);
+
+ if (tbls.size() != 1)
+ throw new IgniteSQLException("Failed to determine target table", IgniteQueryErrorCode.TABLE_NOT_FOUND);
+
+ return tbls.iterator().next();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlBatchSender.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlBatchSender.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlBatchSender.java
new file mode 100644
index 0000000..a4a60c3
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlBatchSender.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.util.typedef.F;
+
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import static org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode.createJdbcSqlException;
+
+/**
+ * Batch sender class.
+ */
+public class DmlBatchSender {
+ /** Cache context. */
+ private final GridCacheContext cctx;
+
+ /** Batch size. */
+ private final int size;
+
+ /** Batches. */
+ private final Map<UUID, Map<Object, EntryProcessor<Object, Object, Boolean>>> batches = new HashMap<>();
+
+ /** Result count. */
+ private long updateCnt;
+
+ /** Failed keys. */
+ private List<Object> failedKeys;
+
+ /** Exception. */
+ private SQLException err;
+
+ /**
+ * Constructor.
+ *
+ * @param cctx Cache context.
+ * @param size Batch.
+ */
+ public DmlBatchSender(GridCacheContext cctx, int size) {
+ this.cctx = cctx;
+ this.size = size;
+ }
+
+ /**
+ * Add entry to batch.
+ *
+ * @param key Key.
+ * @param proc Processor.
+ */
+ public void add(Object key, EntryProcessor<Object, Object, Boolean> proc) throws IgniteCheckedException {
+ ClusterNode node = cctx.affinity().primaryByKey(key, AffinityTopologyVersion.NONE);
+
+ if (node == null)
+ throw new IgniteCheckedException("Failed to map key to node.");
+
+ UUID nodeId = node.id();
+
+ Map<Object, EntryProcessor<Object, Object, Boolean>> batch = batches.get(nodeId);
+
+ if (batch == null) {
+ batch = new HashMap<>();
+
+ batches.put(nodeId, batch);
+ }
+
+ batch.put(key, proc);
+
+ if (batch.size() >= size) {
+ sendBatch(batch);
+
+ batch.clear();
+ }
+ }
+
+ /**
+ * Flush any remaining entries.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void flush() throws IgniteCheckedException {
+ for (Map<Object, EntryProcessor<Object, Object, Boolean>> batch : batches.values()) {
+ if (!batch.isEmpty())
+ sendBatch(batch);
+ }
+ }
+
+ /**
+ * @return Update count.
+ */
+ public long updateCount() {
+ return updateCnt;
+ }
+
+ /**
+ * @return Failed keys.
+ */
+ public List<Object> failedKeys() {
+ return failedKeys != null ? failedKeys : Collections.emptyList();
+ }
+
+ /**
+ * @return Error.
+ */
+ public SQLException error() {
+ return err;
+ }
+
+ /**
+ * Send the batch.
+ *
+ * @param batch Batch.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void sendBatch(Map<Object, EntryProcessor<Object, Object, Boolean>> batch)
+ throws IgniteCheckedException {
+ DmlPageProcessingResult pageRes = processPage(cctx, batch);
+
+ updateCnt += pageRes.count();
+
+ if (failedKeys == null)
+ failedKeys = new ArrayList<>();
+
+ failedKeys.addAll(F.asList(pageRes.errorKeys()));
+
+ if (pageRes.error() != null) {
+ if (err == null)
+ err = error();
+ else
+ err.setNextException(error());
+ }
+ }
+
+ /**
+ * Execute given entry processors and collect errors, if any.
+ * @param cctx Cache context.
+ * @param rows Rows to process.
+ * @return Triple [number of rows actually changed; keys that failed to update (duplicates or concurrently
+ * updated ones); chain of exceptions for all keys whose processing resulted in error, or null for no errors].
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings({"unchecked", "ConstantConditions"})
+ private static DmlPageProcessingResult processPage(GridCacheContext cctx,
+ Map<Object, EntryProcessor<Object, Object, Boolean>> rows) throws IgniteCheckedException {
+ Map<Object, EntryProcessorResult<Boolean>> res = cctx.cache().invokeAll(rows);
+
+ if (F.isEmpty(res))
+ return new DmlPageProcessingResult(rows.size(), null, null);
+
+ DmlPageProcessingErrorResult splitRes = splitErrors(res);
+
+ int keysCnt = splitRes.errorKeys().length;
+
+ return new DmlPageProcessingResult(rows.size() - keysCnt - splitRes.errorCount(), splitRes.errorKeys(),
+ splitRes.error());
+ }
+
+ /**
+ * Process errors of entry processor - split the keys into duplicated/concurrently modified and those whose
+ * processing yielded an exception.
+ *
+ * @param res Result of {@link GridCacheAdapter#invokeAll)}
+ * @return pair [array of duplicated/concurrently modified keys, SQL exception for erroneous keys] (exception is
+ * null if all keys are duplicates/concurrently modified ones).
+ */
+ private static DmlPageProcessingErrorResult splitErrors(Map<Object, EntryProcessorResult<Boolean>> res) {
+ Set<Object> errKeys = new LinkedHashSet<>(res.keySet());
+
+ SQLException currSqlEx = null;
+
+ SQLException firstSqlEx = null;
+
+ int errors = 0;
+
+ // Let's form a chain of SQL exceptions
+ for (Map.Entry<Object, EntryProcessorResult<Boolean>> e : res.entrySet()) {
+ try {
+ e.getValue().get();
+ }
+ catch (EntryProcessorException ex) {
+ SQLException next = createJdbcSqlException("Failed to process key '" + e.getKey() + '\'',
+ IgniteQueryErrorCode.ENTRY_PROCESSING);
+
+ next.initCause(ex);
+
+ if (currSqlEx != null)
+ currSqlEx.setNextException(next);
+ else
+ firstSqlEx = next;
+
+ currSqlEx = next;
+
+ errKeys.remove(e.getKey());
+
+ errors++;
+ }
+ }
+
+ return new DmlPageProcessingErrorResult(errKeys.toArray(), firstSqlEx, errors);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlDistributedPlanInfo.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlDistributedPlanInfo.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlDistributedPlanInfo.java
new file mode 100644
index 0000000..44c6e22
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlDistributedPlanInfo.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import java.util.List;
+
+/**
+ * Additional information about distributed update plan.
+ */
+public final class DmlDistributedPlanInfo {
+ /** Whether update involves only replicated caches. */
+ private final boolean replicatedOnly;
+
+ /** Identifiers of caches involved in update (used for cluster nodes mapping). */
+ private final List<Integer> cacheIds;
+
+ /**
+ * Constructor.
+ *
+ * @param replicatedOnly Whether all caches are replicated.
+ * @param cacheIds List of cache identifiers.
+ */
+ public DmlDistributedPlanInfo(boolean replicatedOnly, List<Integer> cacheIds) {
+ this.replicatedOnly = replicatedOnly;
+ this.cacheIds = cacheIds;
+ }
+
+ /**
+ * @return {@code true} in case all involved caches are replicated.
+ */
+ public boolean isReplicatedOnly() {
+ return replicatedOnly;
+ }
+
+ /**
+ * @return cache identifiers.
+ */
+ public List<Integer> getCacheIds() {
+ return cacheIds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingErrorResult.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingErrorResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingErrorResult.java
new file mode 100644
index 0000000..02e7359
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingErrorResult.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import javax.cache.processor.EntryProcessor;
+import java.sql.SQLException;
+
+/**
+ * Result of splitting keys whose processing resulted into an exception from those skipped by
+ * logic of {@link EntryProcessor}s (most likely INSERT duplicates, or UPDATE/DELETE keys whose values
+ * had been modified concurrently), counting and collecting entry processor exceptions.
+ */
+public final class DmlPageProcessingErrorResult {
+ /** Keys that failed to be processed by {@link EntryProcessor} (not due to an exception). */
+ private final Object[] errKeys;
+
+ /** Number of entries whose processing resulted into an exception. */
+ private final int cnt;
+
+ /** Chain of exceptions corresponding to failed keys. Null if no keys yielded an exception. */
+ private final SQLException ex;
+
+ /** */
+ @SuppressWarnings("ConstantConditions")
+ public DmlPageProcessingErrorResult(@NotNull Object[] errKeys, SQLException ex, int exCnt) {
+ errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY);
+ // When exceptions count must be zero, exceptions chain must be not null, and vice versa.
+ assert exCnt == 0 ^ ex != null;
+
+ this.errKeys = errKeys;
+ this.cnt = exCnt;
+ this.ex = ex;
+ }
+
+ /**
+ * @return Number of entries whose processing resulted into an exception.
+ */
+ public int errorCount() {
+ return cnt;
+ }
+
+ /**
+ * @return Error keys.
+ */
+ public Object[] errorKeys() {
+ return errKeys;
+ }
+
+ /**
+ * @return Error.
+ */
+ @Nullable
+ public SQLException error() {
+ return ex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingResult.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingResult.java
new file mode 100644
index 0000000..f2db3a7
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlPageProcessingResult.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import java.sql.SQLException;
+
+/**
+ * Result of processing an individual page with {@link IgniteCache#invokeAll} including error details, if any.
+ */
+public final class DmlPageProcessingResult {
+ /** Number of successfully processed items. */
+ private final long cnt;
+
+ /** Keys that failed to be updated or deleted due to concurrent modification of values. */
+ private final Object[] errKeys;
+
+ /** Chain of exceptions corresponding to failed keys. Null if no keys yielded an exception. */
+ private final SQLException ex;
+
+ /** */
+ @SuppressWarnings("ConstantConditions")
+ public DmlPageProcessingResult(long cnt, Object[] errKeys, @Nullable SQLException ex) {
+ this.cnt = cnt;
+ this.errKeys = U.firstNotNull(errKeys, X.EMPTY_OBJECT_ARRAY);
+ this.ex = ex;
+ }
+
+ /**
+ * @return Number of successfully processed items.
+ */
+ public long count() {
+ return cnt;
+ }
+
+ /**
+ * @return Error keys.
+ */
+ public Object[] errorKeys() {
+ return errKeys;
+ }
+
+ /**
+ * @return Error.
+ */
+ @Nullable public SQLException error() {
+ return ex;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
new file mode 100644
index 0000000..6621fc2
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.processors.query.h2.H2Utils;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.h2.util.DateTimeUtils;
+import org.h2.util.LocalDateTimeUtils;
+import org.h2.value.Value;
+import org.h2.value.ValueDate;
+import org.h2.value.ValueTime;
+import org.h2.value.ValueTimestamp;
+
+import java.lang.reflect.Array;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Date;
+
+/**
+ * DML utility methods.
+ */
+public class DmlUtils {
+ /**
+ * Convert value to column's expected type by means of H2.
+ *
+ * @param val Source value.
+ * @param desc Row descriptor.
+ * @param expCls Expected value class.
+ * @param type Expected column type to convert to.
+ * @return Converted object.
+ * @throws IgniteCheckedException if failed.
+ */
+ @SuppressWarnings({"ConstantConditions", "SuspiciousSystemArraycopy"})
+ public static Object convert(Object val, GridH2RowDescriptor desc, Class<?> expCls, int type)
+ throws IgniteCheckedException {
+ if (val == null)
+ return null;
+
+ Class<?> currCls = val.getClass();
+
+ try {
+ if (val instanceof Date && currCls != Date.class && expCls == Date.class) {
+ // H2 thinks that java.util.Date is always a Timestamp, while binary marshaller expects
+ // precise Date instance. Let's satisfy it.
+ return new Date(((Date) val).getTime());
+ }
+
+ // User-given UUID is always serialized by H2 to byte array, so we have to deserialize manually
+ if (type == Value.UUID && currCls == byte[].class)
+ return U.unmarshal(desc.context().marshaller(), (byte[]) val,
+ U.resolveClassLoader(desc.context().gridConfig()));
+
+ if (LocalDateTimeUtils.isJava8DateApiPresent()) {
+ if (val instanceof Timestamp && LocalDateTimeUtils.isLocalDateTime(expCls))
+ return LocalDateTimeUtils.valueToLocalDateTime(ValueTimestamp.get((Timestamp) val));
+
+ if (val instanceof Date && LocalDateTimeUtils.isLocalDate(expCls))
+ return LocalDateTimeUtils.valueToLocalDate(ValueDate.fromDateValue(
+ DateTimeUtils.dateValueFromDate(((Date) val).getTime())));
+
+ if (val instanceof Time && LocalDateTimeUtils.isLocalTime(expCls))
+ return LocalDateTimeUtils.valueToLocalTime(ValueTime.get((Time) val));
+ }
+
+ // We have to convert arrays of reference types manually -
+ // see https://issues.apache.org/jira/browse/IGNITE-4327
+ // Still, we only can convert from Object[] to something more precise.
+ if (type == Value.ARRAY && currCls != expCls) {
+ if (currCls != Object[].class)
+ throw new IgniteCheckedException("Unexpected array type - only conversion from Object[] " +
+ "is assumed");
+
+ // Why would otherwise type be Value.ARRAY?
+ assert expCls.isArray();
+
+ Object[] curr = (Object[]) val;
+
+ Object newArr = Array.newInstance(expCls.getComponentType(), curr.length);
+
+ System.arraycopy(curr, 0, newArr, 0, curr.length);
+
+ return newArr;
+ }
+
+ return H2Utils.convert(val, desc, type);
+ }
+ catch (Exception e) {
+ throw new IgniteSQLException("Value conversion failed [from=" + currCls.getName() + ", to=" +
+ expCls.getName() +']', IgniteQueryErrorCode.CONVERSION_FAILED, e);
+ }
+ }
+
+ /**
+ * Private constructor.
+ */
+ private DmlUtils() {
+ // No-op.
+ }
+}
[47/50] [abbrv] ignite git commit: IGNITE-7244 Added missing
README.txt for development utils
Posted by sb...@apache.org.
IGNITE-7244 Added missing README.txt for development utils
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6e8cfe32
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6e8cfe32
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6e8cfe32
Branch: refs/heads/ignite-zk
Commit: 6e8cfe324fc7ea69a2b5477ac3187d333aef3134
Parents: 7782d38
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Dec 19 15:33:03 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Dec 19 15:33:03 2017 +0300
----------------------------------------------------------------------
modules/dev-utils/README.txt | 5 +++++
.../java/org/apache/ignite/development/utils/package-info.java | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e8cfe32/modules/dev-utils/README.txt
----------------------------------------------------------------------
diff --git a/modules/dev-utils/README.txt b/modules/dev-utils/README.txt
new file mode 100644
index 0000000..f839c4f
--- /dev/null
+++ b/modules/dev-utils/README.txt
@@ -0,0 +1,5 @@
+Apache Ignite Development Utils
+------------------------
+
+Special module contains Ignite development utilities. These utilities may be useful when analyzing PDS artifacts,
+such as WAL or page storage, in field deployments.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6e8cfe32/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/package-info.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/package-info.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/package-info.java
index d9018ed..b3119dd 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/package-info.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/package-info.java
@@ -17,6 +17,6 @@
/**
* <!-- Package description. -->
- * Contains converter for WAL logs.
+ * Contains development utilities for Ignite useful for PDS artifacts analysis.
*/
package org.apache.ignite.development.utils;
\ No newline at end of file
[37/50] [abbrv] ignite git commit: IGNITE-4862: Fixed NPE when
reading data from IGFS. This closes #1706.
Posted by sb...@apache.org.
IGNITE-4862: Fixed NPE when reading data from IGFS. This closes #1706.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5cd3e263
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5cd3e263
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5cd3e263
Branch: refs/heads/ignite-zk
Commit: 5cd3e263b9ca1464044c1eec24fc642873a881ab
Parents: edc66af
Author: Ilya Kasnacheev <il...@gmail.com>
Authored: Mon Dec 18 15:58:10 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Mon Dec 18 15:58:10 2017 +0300
----------------------------------------------------------------------
.../processors/igfs/IgfsDataManager.java | 4 +-
...zySecondaryFileSystemPositionedReadable.java | 15 +++++--
...fsSecondaryFileSystemPositionedReadable.java | 41 ++++++++++++--------
3 files changed, 38 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5cd3e263/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index 12765df..90b5e9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -419,9 +419,11 @@ public class IgfsDataManager extends IgfsManager {
int read = 0;
try {
+ int r;
+
// Delegate to the secondary file system.
while (read < blockSize) {
- int r = secReader.read(pos + read, res, read, blockSize - read);
+ r = secReader.read(pos + read, res, read, blockSize - read);
if (r < 0)
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5cd3e263/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
index 0a57c34..bbb5420 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsLazySecondaryFileSystemPositionedReadable.java
@@ -37,6 +37,9 @@ public class IgfsLazySecondaryFileSystemPositionedReadable implements IgfsSecond
/** Buffer size. */
private final int bufSize;
+ /** Synchronization mutex. */
+ private final Object mux = new Object();
+
/** Target stream. */
private IgfsSecondaryFileSystemPositionedReadable target;
@@ -58,16 +61,20 @@ public class IgfsLazySecondaryFileSystemPositionedReadable implements IgfsSecond
/** {@inheritDoc} */
@Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
- if (target == null)
- target = fs.open(path, bufSize);
+ synchronized (mux) {
+ if (target == null)
+ target = fs.open(path, bufSize);
+ }
return target.read(pos, buf, off, len);
}
/** {@inheritDoc} */
@Override public void close() throws IOException {
- if (target != null)
- target.close();
+ synchronized (mux) {
+ if (target != null)
+ target.close();
+ }
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/5cd3e263/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
index 1a4add5..84f6283 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
@@ -42,6 +42,9 @@ public class HadoopIgfsSecondaryFileSystemPositionedReadable implements IgfsSeco
/** Buffer size. */
private final int bufSize;
+ /** Synchronization mutex. */
+ private final Object mux = new Object();
+
/** Actual input stream. */
private FSDataInputStream in;
@@ -69,34 +72,38 @@ public class HadoopIgfsSecondaryFileSystemPositionedReadable implements IgfsSeco
/** Get input stream. */
private PositionedReadable in() throws IOException {
- if (opened) {
- if (err != null)
- throw err;
- }
- else {
- opened = true;
+ synchronized (mux) {
+ if (opened) {
+ if (err != null)
+ throw err;
+ }
+ else {
+ opened = true;
- try {
- in = fs.open(path, bufSize);
+ try {
+ in = fs.open(path, bufSize);
- if (in == null)
- throw new IOException("Failed to open input stream (file system returned null): " + path);
- }
- catch (IOException e) {
- err = e;
+ if (in == null)
+ throw new IOException("Failed to open input stream (file system returned null): " + path);
+ }
+ catch (IOException e) {
+ err = e;
- throw err;
+ throw err;
+ }
}
- }
- return in;
+ return in;
+ }
}
/**
* Close wrapped input stream in case it was previously opened.
*/
@Override public void close() {
- U.closeQuiet(in);
+ synchronized (mux) {
+ U.closeQuiet(in);
+ }
}
/** {@inheritDoc} */
[34/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk-ce
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk-ce
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/81014550
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/81014550
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/81014550
Branch: refs/heads/ignite-zk
Commit: 8101455034a723eadf94dabe4f5d8f1fc0ae5606
Parents: ed492a4 91be7af
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 18 10:38:07 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 18 10:38:07 2017 +0300
----------------------------------------------------------------------
.../spi/communication/tcp/TcpCommunicationSpi.java | 3 +--
.../TcpCommunicationConnectionCheckFuture.java | 17 ++++++++++++++++-
.../ignite/ml/knn/models/KNNModelFormat.java | 6 +++++-
3 files changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/81014550/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/81014550/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
index 99e1eca,0000000..6cb5622
mode 100644,000000..100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
@@@ -1,461 -1,0 +1,476 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.net.InetSocketAddress;
+import java.nio.channels.SocketChannel;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
++import org.apache.ignite.events.DiscoveryEvent;
++import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteInternalFuture;
++import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.util.GridLeanMap;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.nio.GridNioServer;
+import org.apache.ignite.internal.util.nio.GridNioSession;
+import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jetbrains.annotations.Nullable;
+
++import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
++import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
++
+/**
+ *
+ */
- public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<BitSet> implements IgniteSpiTimeoutObject {
++public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<BitSet> implements IgniteSpiTimeoutObject, GridLocalEventListener {
+ /** Session future. */
+ public static final int SES_FUT_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** */
+ public static final ConnectionKey CONN_CHECK_DUMMY_KEY = new ConnectionKey(null, -1, -1);
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<SingleAddressConnectFuture> connFutDoneUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(SingleAddressConnectFuture.class, "done");
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<MultipleAddressesConnectFuture> connResCntUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(MultipleAddressesConnectFuture.class, "resCnt");
+
+ /** */
+ private final AtomicInteger resCntr = new AtomicInteger();
+
+ /** */
+ private final List<ClusterNode> nodes;
+
+ /** */
+ private volatile ConnectFuture[] futs;
+
+ /** */
+ private final GridNioServer nioSrvr;
+
+ /** */
+ private final TcpCommunicationSpi spi;
+
+ /** */
+ private final IgniteUuid timeoutObjId = IgniteUuid.randomUuid();
+
+ /** */
+ private final BitSet resBitSet;
+
+ /** */
+ private long endTime;
+
+ /** */
+ private final IgniteLogger log;
+
+ /**
+ * @param spi SPI instance.
+ * @param log Logger.
+ * @param nioSrvr NIO server.
+ * @param nodes Nodes to check.
+ */
+ public TcpCommunicationConnectionCheckFuture(TcpCommunicationSpi spi,
+ IgniteLogger log,
+ GridNioServer nioSrvr,
+ List<ClusterNode> nodes)
+ {
+ this.spi = spi;
+ this.log = log;
+ this.nioSrvr = nioSrvr;
+ this.nodes = nodes;
+
+ resBitSet = new BitSet(nodes.size());
+ }
+
++ /** {@inheritDoc} */
++ @Override public void onEvent(Event evt) {
++ assert evt instanceof DiscoveryEvent : evt;
++ assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ;
++
++ }
++
+ /**
+ * @param timeout Connect timeout.
+ */
+ public void init(long timeout) {
+ ConnectFuture[] futs = new ConnectFuture[nodes.size()];
+
+ UUID locId = spi.getSpiContext().localNode().id();
+
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+
+ if (!node.id().equals(locId)) {
+ if (spi.getSpiContext().node(node.id()) == null) {
+ receivedConnectionStatus(i, false);
+
+ continue;
+ }
+
+ Collection<InetSocketAddress> addrs;
+
+ try {
+ addrs = spi.nodeAddresses(node, false);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to get node addresses: " + node, e);
+
+ receivedConnectionStatus(i, false);
+
+ continue;
+ }
+
+ if (addrs.size() == 1) {
+ SingleAddressConnectFuture fut = new SingleAddressConnectFuture(i);
+
+ fut.init(addrs.iterator().next());
+
+ futs[i] = fut;
+ }
+ else {
+ MultipleAddressesConnectFuture fut = new MultipleAddressesConnectFuture(i);
+
+ fut.init(addrs);
+ }
+ }
+ else
+ receivedConnectionStatus(i, true);
+ }
+
+ this.futs = futs;
+
++ spi.getSpiContext().addLocalEventListener(this, EVT_NODE_LEFT, EVT_NODE_FAILED);
++
+ if (!isDone()) {
+ endTime = System.currentTimeMillis() - timeout;
+
+ spi.getSpiContext().addTimeoutObject(this);
+ }
+ }
+
+ /**
+ * @param idx Node index.
+ * @param res Success flag.
+ */
+ private void receivedConnectionStatus(int idx, boolean res) {
+ assert resCntr.get() < nodes.size();
+
+ synchronized (resBitSet) {
+ resBitSet.set(idx, res);
+ }
+
+ if (resCntr.incrementAndGet() == nodes.size())
+ onDone(resBitSet);
+ }
+
+ /**
+ * @param nodeIdx Node index.
+ * @return Node ID.
+ */
+ private UUID nodeId(int nodeIdx) {
+ return nodes.get(nodeIdx).id();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid id() {
+ return timeoutObjId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return endTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ if (!isDone())
+ return;
+
+ ConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onTimeout();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable BitSet res, @Nullable Throwable err) {
+ if (super.onDone(res, err)) {
+ spi.getSpiContext().removeTimeoutObject(this);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ *
+ */
+ private interface ConnectFuture {
+ /**
+ *
+ */
+ void onTimeout();
+ }
+
+ /**
+ *
+ */
+ private class SingleAddressConnectFuture implements TcpCommunicationNodeConnectionCheckFuture, ConnectFuture {
+ /** */
+ final int nodeIdx;
+
+ /** */
+ volatile int done;
+
+ /** */
+ Map<Integer, Object> sesMeta;
+
+ /** */
+ private SocketChannel ch;
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ SingleAddressConnectFuture(int nodeIdx) {
+ this.nodeIdx = nodeIdx;
+ }
+
+ /**
+ * @param addr Node address.
+ */
+ public void init(InetSocketAddress addr) {
+ boolean connect;
+
+ try {
+ ch = SocketChannel.open();
+
+ ch.configureBlocking(false);
+
+ ch.socket().setTcpNoDelay(true);
+ ch.socket().setKeepAlive(false);
+
+ connect = ch.connect(addr);
+ }
+ catch (Exception e) {
+ finish(false);
+
+ return;
+ }
+
+ if (!connect) {
+ sesMeta = new GridLeanMap<>(3);
+
+ // Set dummy key to identify connection-check outgoing connection.
+ sesMeta.put(TcpCommunicationSpi.CONN_IDX_META, CONN_CHECK_DUMMY_KEY);
+ sesMeta.put(SES_FUT_META, this);
+
+ nioSrvr.createSession(ch, sesMeta, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
+ @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
+ if (fut.error() != null)
+ finish(false);
+ }
+ });
+ }
+ }
+
+ /**
+ *
+ */
+ void cancel() {
+ if (finish(false))
+ nioSrvr.cancelConnect(ch, sesMeta);
+ }
+
+ /** {@inheritDoc} */
+ public void onTimeout() {
+ cancel();
+ }
+
+ /** {@inheritDoc} */
+ public void onConnected(UUID rmtNodeId) {
+ finish(nodeId(nodeIdx).equals(rmtNodeId));
+ }
+
+ /**
+ * @param res Result.
+ * @return {@code True} if result was set by this call.
+ */
+ boolean finish(boolean res) {
+ if (connFutDoneUpdater.compareAndSet(this, 0, 1)) {
+ onStatusReceived(res);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param res Result.
+ */
+ void onStatusReceived(boolean res) {
+ receivedConnectionStatus(nodeIdx, res);
+ }
+ }
+
+ /**
+ *
+ */
+ private class MultipleAddressesConnectFuture implements ConnectFuture {
+ /** */
+ volatile int resCnt;
+
+ /** */
+ volatile SingleAddressConnectFuture[] futs;
+
+ /** */
+ final int nodeIdx;
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ MultipleAddressesConnectFuture(int nodeIdx) {
+ this.nodeIdx = nodeIdx;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onTimeout();
+ }
+ }
+
+ /**
+ * @param addrs Node addresses.
+ */
+ void init(Collection<InetSocketAddress> addrs) {
+ SingleAddressConnectFuture[] futs = new SingleAddressConnectFuture[addrs.size()];
+
+ int idx = 0;
+
+ for (InetSocketAddress addr : addrs) {
+ SingleAddressConnectFuture fut = new SingleAddressConnectFuture(nodeIdx) {
+ @Override void onStatusReceived(boolean res) {
+ receivedAddressStatus(res);
+ }
+ };
+
+ fut.init(addr);
+
+ futs[idx++] = fut;
+
+ if (done())
+ return;
+ }
+
+ this.futs = futs;
+
+ // Close race.
+ if (done())
+ cancelFutures();
+ }
+
+ /**
+ * @return {@code True}
+ */
+ private boolean done() {
+ int resCnt0 = resCnt;
+
+ return resCnt0 == Integer.MAX_VALUE || resCnt0 == futs.length;
+ }
+
+ /**
+ *
+ */
+ private void cancelFutures() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ if (futs != null) {
+ for (int i = 0; i < futs.length; i++) {
+ SingleAddressConnectFuture fut = futs[i];
+
+ fut.cancel();
+ }
+ }
+ }
+
+ /**
+ * @param res Result.
+ */
+ void receivedAddressStatus(boolean res) {
+ if (res) {
+ for (;;) {
+ int resCnt0 = resCnt;
+
+ if (resCnt0 == Integer.MAX_VALUE)
+ return;
+
+ if (connResCntUpdater.compareAndSet(this, resCnt0, Integer.MAX_VALUE)) {
+ receivedConnectionStatus(nodeIdx, true);
+
+ cancelFutures(); // Cancel others connects if they are still in progress.
+
+ return;
+ }
+ }
+ }
+ else {
+ for (;;) {
+ int resCnt0 = resCnt;
+
+ if (resCnt0 == Integer.MAX_VALUE)
+ return;
+
+ int resCnt1 = resCnt0 + 1;
+
+ if (connResCntUpdater.compareAndSet(this, resCnt0, resCnt1)) {
+ if (resCnt1 == futs.length)
+ receivedConnectionStatus(nodeIdx, false);
+
+ return;
+ }
+ }
+ }
+ }
+ }
+}
[13/50] [abbrv] ignite git commit: IGNITE-6920 Fixed rhel detection
in web console direct-install.
Posted by sb...@apache.org.
IGNITE-6920 Fixed rhel detection in web console direct-install.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/33258762
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/33258762
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/33258762
Branch: refs/heads/ignite-zk
Commit: 33258762a92214bda691b7f3fa5d93ad3aee2e21
Parents: c9d6dd5
Author: Andrey Novikov <an...@gridgain.com>
Authored: Thu Dec 14 11:32:11 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Thu Dec 14 11:32:11 2017 +0700
----------------------------------------------------------------------
modules/web-console/DEVNOTES.txt | 2 +-
modules/web-console/pom.xml | 111 ++++++++++++++++++++++++++++++++--
2 files changed, 106 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/33258762/modules/web-console/DEVNOTES.txt
----------------------------------------------------------------------
diff --git a/modules/web-console/DEVNOTES.txt b/modules/web-console/DEVNOTES.txt
index 6dbb15a..dfb017e 100644
--- a/modules/web-console/DEVNOTES.txt
+++ b/modules/web-console/DEVNOTES.txt
@@ -35,6 +35,6 @@ How to migrate model:
Ignite Web Console Direct-Install Maven Build Instructions
==========================================================
To build direct-install archive from sources run following command in Ignite project root folder:
-"mvn clean package -pl :ignite-web-agent,:ignite-web-console -am -P web-console -DskipTests=true"
+"mvn clean package -pl :ignite-web-agent,:ignite-web-console -am -P web-console -DskipTests=true -DskipClientDocs -Dmaven.javadoc.skip=true"
Assembled archive can be found here: `/modules/web-console/target/ignite-web-console-direct-install-*.zip`.
http://git-wip-us.apache.org/repos/asf/ignite/blob/33258762/modules/web-console/pom.xml
----------------------------------------------------------------------
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index d5b72bf..1ef1992 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -88,11 +88,25 @@
</execution>
<execution>
+ <id>download dependencies for backend</id>
+ <goals>
+ <goal>npm</goal>
+ </goals>
+
+ <configuration>
+ <workingDirectory>backend</workingDirectory>
+ <arguments>install --no-optional</arguments>
+ </configuration>
+ </execution>
+
+ <execution>
<id>build frontend</id>
<goals>
<goal>npm</goal>
</goals>
+ <phase>compile</phase>
+
<configuration>
<workingDirectory>frontend</workingDirectory>
<arguments>run build</arguments>
@@ -103,26 +117,111 @@
</execution>
<execution>
- <id>download dependencies for backend</id>
+ <id>build backend</id>
<goals>
<goal>npm</goal>
</goals>
+ <phase>compile</phase>
+
<configuration>
<workingDirectory>backend</workingDirectory>
- <arguments>install --no-optional</arguments>
+ <arguments>run build</arguments>
</configuration>
</execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-clean-plugin</artifactId>
+ <version>2.5</version>
+ <executions>
<execution>
- <id>build backend</id>
+ <id>clean-frontend-build</id>
<goals>
- <goal>npm</goal>
+ <goal>clean</goal>
+ </goals>
+ <phase>process-resources</phase>
+ <configuration>
+ <excludeDefaultDirectories>true</excludeDefaultDirectories>
+ <filesets>
+ <fileset>
+ <directory>${project.basedir}/frontend/build</directory>
+ </fileset>
+ </filesets>
+ </configuration>
+ </execution>
+
+ <execution>
+ <id>clean-backend-build</id>
+ <goals>
+ <goal>clean</goal>
</goals>
+ <phase>process-resources</phase>
+ <configuration>
+ <excludeDefaultDirectories>true</excludeDefaultDirectories>
+ <filesets>
+ <fileset>
+ <directory>${project.basedir}/backend/build</directory>
+ </fileset>
+ </filesets>
+ </configuration>
+ </execution>
+ <execution>
+ <id>remove-old-getos</id>
+ <goals>
+ <goal>clean</goal>
+ </goals>
+ <phase>process-resources</phase>
<configuration>
- <workingDirectory>backend</workingDirectory>
- <arguments>run build</arguments>
+ <excludeDefaultDirectories>true</excludeDefaultDirectories>
+ <filesets>
+ <fileset>
+ <directory>${project.basedir}/backend/node_modules/mongodb-download/node_modules</directory>
+ </fileset>
+ </filesets>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.7</version>
+ <executions>
+ <execution>
+ <id>fixed-rhel-detection</id>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <phase>process-resources</phase>
+ <configuration>
+ <target>
+ <replace dir="${basedir}/backend/node_modules/mongodb-download/built"
+ token="/rhel/" value="/Red Hat Linux/i.test(os.dist) || /rhel/">
+ <include name="mongodb-download.js"/>
+ </replace>
+ </target>
+ </configuration>
+ </execution>
+
+ <execution>
+ <id>fixed-download-url</id>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <phase>process-resources</phase>
+ <configuration>
+ <target>
+ <replace dir="${basedir}/backend/node_modules/mongodb-download/built"
+ token="http://downloads.mongodb.org" value="https://fastdl.mongodb.org">
+ <include name="mongodb-download.js"/>
+ </replace>
+ </target>
</configuration>
</execution>
</executions>
[43/50] [abbrv] ignite git commit: IGNITE-4490: SQL: avoid querying
H2 for INSERT and MERGE when it is not needed. This closes #1387.
Posted by sb...@apache.org.
IGNITE-4490: SQL: avoid querying H2 for INSERT and MERGE when it is not needed. This closes #1387.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ec290234
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ec290234
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ec290234
Branch: refs/heads/ignite-zk
Commit: ec290234bf7b54c5549bd7aa0d99b36d3d0638ac
Parents: 8f327a1
Author: Alexander Paschenko <al...@gmail.com>
Authored: Tue Dec 19 11:11:11 2017 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Tue Dec 19 11:11:11 2017 +0300
----------------------------------------------------------------------
.../ignite/jdbc/JdbcErrorsAbstractSelfTest.java | 2 +-
.../ignite/jdbc/thin/JdbcThinBatchSelfTest.java | 5 +-
.../query/h2/DmlStatementsProcessor.java | 53 +++++-----
.../processors/query/h2/dml/DmlArgument.java | 31 ++++++
.../processors/query/h2/dml/DmlArguments.java | 104 +++++++++++++++++++
.../processors/query/h2/dml/DmlAstUtils.java | 14 ++-
.../processors/query/h2/dml/DmlUtils.java | 19 ++--
.../processors/query/h2/dml/FastUpdate.java | 91 ++--------------
.../query/h2/dml/FastUpdateArgument.java | 27 -----
.../processors/query/h2/dml/UpdatePlan.java | 80 +++++++++++---
.../query/h2/dml/UpdatePlanBuilder.java | 37 ++++++-
11 files changed, 304 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
index 952baa5..fb96f31 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/JdbcErrorsAbstractSelfTest.java
@@ -107,7 +107,7 @@ public abstract class JdbcErrorsAbstractSelfTest extends GridCommonAbstractTest
public void testDmlErrors() throws SQLException {
checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, null)", "22004");
- checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, 'zzz')", "50000");
+ checkErrorState("INSERT INTO \"test\".INTEGER(_key, _val) values(1, 'zzz')", "0700B");
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinBatchSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinBatchSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinBatchSelfTest.java
index 5e2e39e..8609615 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinBatchSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinBatchSelfTest.java
@@ -260,7 +260,8 @@ public class JdbcThinBatchSelfTest extends JdbcThinAbstractDmlStatementSelfTest
pstmt.executeBatch();
fail("BatchUpdateException must be thrown");
- } catch(BatchUpdateException e) {
+ }
+ catch(BatchUpdateException e) {
int [] updCnts = e.getUpdateCounts();
assertEquals("Invalid update counts size", BATCH_SIZE, updCnts.length);
@@ -268,7 +269,7 @@ public class JdbcThinBatchSelfTest extends JdbcThinAbstractDmlStatementSelfTest
for (int i = 0; i < BATCH_SIZE; ++i)
assertEquals("Invalid update count",1, updCnts[i]);
- if (!e.getMessage().contains("Failed to execute SQL query.")) {
+ if (!e.getMessage().contains("Value conversion failed")) {
log.error("Invalid exception: ", e);
fail();
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
index 243d1dc..9a6b0af 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DmlStatementsProcessor.java
@@ -32,7 +32,6 @@ import java.util.concurrent.TimeUnit;
import javax.cache.processor.EntryProcessor;
import javax.cache.processor.EntryProcessorException;
import javax.cache.processor.MutableEntry;
-
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.IgniteException;
@@ -51,7 +50,6 @@ import org.apache.ignite.internal.processors.query.GridQueryFieldsResultAdapter;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.h2.dml.DmlBatchSender;
import org.apache.ignite.internal.processors.query.h2.dml.DmlDistributedPlanInfo;
-import org.apache.ignite.internal.processors.query.h2.dml.FastUpdate;
import org.apache.ignite.internal.processors.query.h2.dml.UpdateMode;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlan;
import org.apache.ignite.internal.processors.query.h2.dml.UpdatePlanBuilder;
@@ -167,7 +165,7 @@ public class DmlStatementsProcessor {
UpdateResult r;
try {
- r = executeUpdateStatement(schemaName, cctx, conn, prepared, fieldsQry, loc, filters, cancel, errKeys);
+ r = executeUpdateStatement(schemaName, cctx, conn, prepared, fieldsQry, loc, filters, cancel);
}
finally {
cctx.operationContextPerCall(opCtx);
@@ -247,15 +245,13 @@ public class DmlStatementsProcessor {
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
- long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, Object[] args)
+ long streamUpdateQuery(IgniteDataStreamer streamer, PreparedStatement stmt, final Object[] args)
throws IgniteCheckedException {
- args = U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY);
-
Prepared p = GridSqlQueryParser.prepared(stmt);
assert p != null;
- UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, true, idx, null, null, null);
+ final UpdatePlan plan = UpdatePlanBuilder.planForStatement(p, true, idx, null, null, null);
if (!F.eq(streamer.cacheName(), plan.cacheContext().name()))
throw new IgniteSQLException("Cross cache streaming is not supported, please specify cache explicitly" +
@@ -270,14 +266,22 @@ public class DmlStatementsProcessor {
final ArrayList<List<?>> data = new ArrayList<>(plan.rowCount());
- final GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()), plan.selectQuery(),
- F.asList(args), null, false, 0, null);
-
QueryCursorImpl<List<?>> stepCur = new QueryCursorImpl<>(new Iterable<List<?>>() {
@Override public Iterator<List<?>> iterator() {
try {
- return new GridQueryCacheObjectsIterator(res.iterator(), idx.objectContext(),
- cctx.keepBinary());
+ Iterator<List<?>> it;
+
+ if (!F.isEmpty(plan.selectQuery())) {
+ GridQueryFieldsResult res = idx.queryLocalSqlFields(idx.schema(cctx.name()),
+ plan.selectQuery(), F.asList(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)),
+ null, false, 0, null);
+
+ it = res.iterator();
+ }
+ else
+ it = plan.createRows(U.firstNotNull(args, X.EMPTY_OBJECT_ARRAY)).iterator();
+
+ return new GridQueryCacheObjectsIterator(it, idx.objectContext(), cctx.keepBinary());
}
catch (IgniteCheckedException e) {
throw new IgniteException(e);
@@ -329,27 +333,23 @@ public class DmlStatementsProcessor {
* @param loc Local query flag.
* @param filters Cache name and key filter.
* @param cancel Query cancel state holder.
- * @param failedKeys Keys to restrict UPDATE and DELETE operations with. Null or empty array means no restriction.
* @return Pair [number of successfully processed items; keys that have failed to be processed]
* @throws IgniteCheckedException if failed.
*/
@SuppressWarnings({"ConstantConditions", "unchecked"})
private UpdateResult executeUpdateStatement(String schemaName, final GridCacheContext cctx, Connection c,
- Prepared prepared, SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters,
- GridQueryCancel cancel, Object[] failedKeys) throws IgniteCheckedException {
+ Prepared prepared, SqlFieldsQuery fieldsQry, boolean loc, IndexingQueryFilter filters, GridQueryCancel cancel)
+ throws IgniteCheckedException {
int mainCacheId = cctx.cacheId();
Integer errKeysPos = null;
UpdatePlan plan = getPlanForStatement(schemaName, c, prepared, fieldsQry, loc, errKeysPos);
- FastUpdate fastUpdate = plan.fastUpdate();
-
- if (fastUpdate != null) {
- assert F.isEmpty(failedKeys) && errKeysPos == null;
+ UpdateResult fastUpdateRes = plan.processFast(fieldsQry.getArgs());
- return fastUpdate.execute(plan.cacheContext().cache(), fieldsQry.getArgs());
- }
+ if (fastUpdateRes != null)
+ return fastUpdateRes;
if (plan.distributedPlan() != null) {
UpdateResult result = doDistributedUpdate(schemaName, fieldsQry, plan, cancel);
@@ -359,13 +359,13 @@ public class DmlStatementsProcessor {
return result;
}
- assert !F.isEmpty(plan.selectQuery());
-
- QueryCursorImpl<List<?>> cur;
+ Iterable<List<?>> cur;
// Do a two-step query only if locality flag is not set AND if plan's SELECT corresponds to an actual
// sub-query and not some dummy stuff like "select 1, 2, 3;"
if (!loc && !plan.isLocalSubquery()) {
+ assert !F.isEmpty(plan.selectQuery());
+
SqlFieldsQuery newFieldsQry = new SqlFieldsQuery(plan.selectQuery(), fieldsQry.isCollocated())
.setArgs(fieldsQry.getArgs())
.setDistributedJoins(fieldsQry.isDistributedJoins())
@@ -374,9 +374,10 @@ public class DmlStatementsProcessor {
.setPageSize(fieldsQry.getPageSize())
.setTimeout(fieldsQry.getTimeout(), TimeUnit.MILLISECONDS);
- cur = (QueryCursorImpl<List<?>>)idx.queryDistributedSqlFields(schemaName, newFieldsQry, true,
- cancel, mainCacheId, true).get(0);
+ cur = idx.queryDistributedSqlFields(schemaName, newFieldsQry, true, cancel, mainCacheId, true).get(0);
}
+ else if (plan.hasRows())
+ cur = plan.createRows(fieldsQry.getArgs());
else {
final GridQueryFieldsResult res = idx.queryLocalSqlFields(schemaName, plan.selectQuery(),
F.asList(fieldsQry.getArgs()), filters, fieldsQry.isEnforceJoinOrder(), fieldsQry.getTimeout(), cancel);
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArgument.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArgument.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArgument.java
new file mode 100644
index 0000000..b3c3dce
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArgument.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+/**
+ * DML argument
+ */
+public interface DmlArgument {
+ /**
+ * Get argument from parameter list.
+ *
+ * @param params Query input parameters.
+ * @return value.
+ */
+ Object get(Object[] params);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java
new file mode 100644
index 0000000..f5aa332
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlArguments.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * DML arguments factory.
+ */
+public class DmlArguments {
+ /** Operand that always evaluates as {@code null}. */
+ private final static DmlArgument NULL_ARG = new ConstantArgument(null);
+
+ /**
+ * Create argument from AST element.
+ *
+ * @param el Element.
+ * @return DML argument.
+ */
+ public static DmlArgument create(@Nullable GridSqlElement el) {
+ assert el == null ^ (el instanceof GridSqlConst || el instanceof GridSqlParameter);
+
+ if (el == null)
+ return NULL_ARG;
+
+ if (el instanceof GridSqlConst)
+ return new ConstantArgument(((GridSqlConst)el).value().getObject());
+ else
+ return new ParamArgument(((GridSqlParameter)el).index());
+ }
+
+ /**
+ * Private constructor.
+ */
+ private DmlArguments() {
+ // No-op.
+ }
+
+ /**
+ * Value argument.
+ */
+ private static class ConstantArgument implements DmlArgument {
+ /** Value to return. */
+ private final Object val;
+
+ /**
+ * Constructor.
+ *
+ * @param val Value.
+ */
+ private ConstantArgument(Object val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ public Object get(Object[] params) {
+ return val;
+ }
+ }
+
+ /**
+ * Parameter argument.
+ */
+ private static class ParamArgument implements DmlArgument {
+ /** Value to return. */
+ private final int paramIdx;
+
+ /**
+ * Constructor.
+ *
+ * @param paramIdx Parameter index.
+ */
+ private ParamArgument(int paramIdx) {
+ assert paramIdx >= 0;
+
+ this.paramIdx = paramIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object get(Object[] params) {
+ assert params.length > paramIdx;
+
+ return params[paramIdx];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
index 054e708..161ff4a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlAstUtils.java
@@ -83,7 +83,7 @@ public final class DmlAstUtils {
* @param cols Columns to insert values into.
* @param rows Rows to create pseudo-SELECT upon.
* @param subQry Subquery to use rather than rows.
- * @return Subquery or pseudo-SELECT to evaluate inserted expressions.
+ * @return Subquery or pseudo-SELECT to evaluate inserted expressions, or {@code null} no query needs to be run.
*/
public static GridSqlQuery selectForInsertOrMerge(GridSqlColumn[] cols, List<GridSqlElement[]> rows,
GridSqlQuery subQry) {
@@ -98,6 +98,8 @@ public final class DmlAstUtils {
GridSqlArray[] args = new GridSqlArray[cols.length];
+ boolean noQry = true;
+
for (int i = 0; i < cols.length; i++) {
GridSqlArray arr = new GridSqlArray(rows.size());
@@ -121,10 +123,18 @@ public final class DmlAstUtils {
for (GridSqlElement[] row : rows) {
assert cols.length == row.length;
- for (int i = 0; i < row.length; i++)
+ for (int i = 0; i < row.length; i++) {
+ GridSqlElement el = row[i];
+
+ noQry &= (el instanceof GridSqlConst || el instanceof GridSqlParameter);
+
args[i].addChild(row[i]);
+ }
}
+ if (noQry)
+ return null;
+
return sel;
}
else {
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
index 6621fc2..8d4861e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/DmlUtils.java
@@ -17,6 +17,10 @@
package org.apache.ignite.internal.processors.query.h2.dml;
+import java.lang.reflect.Array;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Date;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
@@ -30,11 +34,6 @@ import org.h2.value.ValueDate;
import org.h2.value.ValueTime;
import org.h2.value.ValueTimestamp;
-import java.lang.reflect.Array;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.util.Date;
-
/**
* DML utility methods.
*/
@@ -101,7 +100,15 @@ public class DmlUtils {
return newArr;
}
- return H2Utils.convert(val, desc, type);
+ Object res = H2Utils.convert(val, desc, type);
+
+ if (res instanceof Date && res.getClass() != Date.class && expCls == Date.class) {
+ // We can get a Timestamp instead of Date when converting a String to Date
+ // without query - let's handle this
+ return new Date(((Date) res).getTime());
+ }
+
+ return res;
}
catch (Exception e) {
throw new IgniteSQLException("Value conversion failed [from=" + currCls.getName() + ", to=" +
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
index e662245..dcceff3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
@@ -20,26 +20,21 @@ package org.apache.ignite.internal.processors.query.h2.dml;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.query.h2.UpdateResult;
-import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement;
-import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter;
import org.jetbrains.annotations.Nullable;
/**
* Arguments for fast, query-less UPDATE or DELETE - key and, optionally, value and new value.
*/
public final class FastUpdate {
- /** Operand that always evaluates as {@code null}. */
- private final static FastUpdateArgument NULL_ARG = new ConstantArgument(null);
-
/** Operand to compute key. */
- private final FastUpdateArgument keyArg;
+ private final DmlArgument keyArg;
/** Operand to compute value. */
- private final FastUpdateArgument valArg;
+ private final DmlArgument valArg;
/** Operand to compute new value. */
- private final FastUpdateArgument newValArg;
+ private final DmlArgument newValArg;
/**
* Create fast update instance.
@@ -50,9 +45,9 @@ public final class FastUpdate {
* @return Fast update.
*/
public static FastUpdate create(GridSqlElement key, GridSqlElement val, @Nullable GridSqlElement newVal) {
- FastUpdateArgument keyArg = argument(key);
- FastUpdateArgument valArg = argument(val);
- FastUpdateArgument newValArg = argument(newVal);
+ DmlArgument keyArg = DmlArguments.create(key);
+ DmlArgument valArg = DmlArguments.create(val);
+ DmlArgument newValArg = DmlArguments.create(newVal);
return new FastUpdate(keyArg, valArg, newValArg);
}
@@ -64,7 +59,7 @@ public final class FastUpdate {
* @param valArg Value argument.
* @param newValArg New value argument.
*/
- private FastUpdate(FastUpdateArgument keyArg, FastUpdateArgument valArg, FastUpdateArgument newValArg) {
+ private FastUpdate(DmlArgument keyArg, DmlArgument valArg, DmlArgument newValArg) {
this.keyArg = keyArg;
this.valArg = valArg;
this.newValArg = newValArg;
@@ -80,12 +75,12 @@ public final class FastUpdate {
*/
@SuppressWarnings({"unchecked", "ConstantConditions"})
public UpdateResult execute(GridCacheAdapter cache, Object[] args) throws IgniteCheckedException {
- Object key = keyArg.apply(args);
+ Object key = keyArg.get(args);
assert key != null;
- Object val = valArg.apply(args);
- Object newVal = newValArg.apply(args);
+ Object val = valArg.get(args);
+ Object newVal = newValArg.get(args);
boolean res;
@@ -106,70 +101,4 @@ public final class FastUpdate {
return res ? UpdateResult.ONE : UpdateResult.ZERO;
}
-
- /**
- * Create argument for AST element.
- *
- * @param el Element.
- * @return Argument.
- */
- private static FastUpdateArgument argument(@Nullable GridSqlElement el) {
- assert el == null ^ (el instanceof GridSqlConst || el instanceof GridSqlParameter);
-
- if (el == null)
- return NULL_ARG;
-
- if (el instanceof GridSqlConst)
- return new ConstantArgument(((GridSqlConst)el).value().getObject());
- else
- return new ParamArgument(((GridSqlParameter)el).index());
- }
-
- /**
- * Value argument.
- */
- private static class ConstantArgument implements FastUpdateArgument {
- /** Value to return. */
- private final Object val;
-
- /**
- * Constructor.
- *
- * @param val Value.
- */
- private ConstantArgument(Object val) {
- this.val = val;
- }
-
- /** {@inheritDoc} */
- @Override public Object apply(Object[] arg) throws IgniteCheckedException {
- return val;
- }
- }
-
- /**
- * Parameter argument.
- */
- private static class ParamArgument implements FastUpdateArgument {
- /** Value to return. */
- private final int paramIdx;
-
- /**
- * Constructor.
- *
- * @param paramIdx Parameter index.
- */
- private ParamArgument(int paramIdx) {
- assert paramIdx >= 0;
-
- this.paramIdx = paramIdx;
- }
-
- /** {@inheritDoc} */
- @Override public Object apply(Object[] arg) throws IgniteCheckedException {
- assert arg.length > paramIdx;
-
- return arg[paramIdx];
- }
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArgument.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArgument.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArgument.java
deleted file mode 100644
index dc90fe9..0000000
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArgument.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.query.h2.dml;
-
-import org.apache.ignite.internal.util.lang.GridPlainClosure;
-
-/**
- * Operand for fast UPDATE or DELETE (single item operation that does not involve any SELECT).
- */
-public interface FastUpdateArgument extends GridPlainClosure<Object[], Object> {
- // No-op.
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
index 31dc52d..6a45c3c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
@@ -17,6 +17,10 @@
package org.apache.ignite.internal.processors.query.h2.dml;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.binary.BinaryObjectBuilder;
@@ -26,6 +30,7 @@ import org.apache.ignite.internal.processors.query.GridQueryProperty;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.h2.UpdateResult;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
import org.apache.ignite.internal.util.typedef.F;
@@ -34,10 +39,6 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.h2.table.Column;
import org.jetbrains.annotations.Nullable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT;
/**
@@ -47,7 +48,7 @@ public final class UpdatePlan {
/** Initial statement to drive the rest of the logic. */
private final UpdateMode mode;
- /** Target table to be affected by initial DML statement. */
+ /** to be affected by initial DML statement. */
private final GridH2Table tbl;
/** Column names to set or update. */
@@ -74,6 +75,9 @@ public final class UpdatePlan {
/** Subquery flag - {@code true} if {@link #selectQry} is an actual subquery that retrieves data from some cache. */
private final boolean isLocSubqry;
+ /** Rows for query-less MERGE or INSERT. */
+ private final List<List<DmlArgument>> rows;
+
/** Number of rows in rows based MERGE or INSERT. */
private final int rowsNum;
@@ -96,6 +100,7 @@ public final class UpdatePlan {
* @param valColIdx value column index.
* @param selectQry Select query.
* @param isLocSubqry Local subquery flag.
+ * @param rows Rows for query-less INSERT or MERGE.
* @param rowsNum Rows number.
* @param fastUpdate Fast update (if any).
* @param distributed Distributed plan (if any)
@@ -111,12 +116,14 @@ public final class UpdatePlan {
int valColIdx,
String selectQry,
boolean isLocSubqry,
+ List<List<DmlArgument>> rows,
int rowsNum,
@Nullable FastUpdate fastUpdate,
@Nullable DmlDistributedPlanInfo distributed
) {
this.colNames = colNames;
this.colTypes = colTypes;
+ this.rows = rows;
this.rowsNum = rowsNum;
assert mode != null;
@@ -161,6 +168,7 @@ public final class UpdatePlan {
-1,
selectQry,
false,
+ null,
0,
fastUpdate,
distributed
@@ -339,6 +347,61 @@ public final class UpdatePlan {
}
/**
+ * Process fast DML operation if possible.
+ *
+ * @param args QUery arguments.
+ * @return Update result or {@code null} if fast update is not applicable for plan.
+ * @throws IgniteCheckedException If failed.
+ */
+ public UpdateResult processFast(Object[] args) throws IgniteCheckedException {
+ if (fastUpdate != null)
+ return fastUpdate.execute(cacheContext().cache(), args);
+
+ return null;
+ }
+
+ /**
+ * @return {@code True} if predefined rows exist.
+ */
+ public boolean hasRows() {
+ return !F.isEmpty(rows);
+ }
+
+ /**
+ * Extract rows from plan without performing any query.
+ * @param args Original query arguments.
+ * @return Rows from plan.
+ * @throws IgniteCheckedException if failed.
+ */
+ public List<List<?>> createRows(Object[] args) throws IgniteCheckedException {
+ assert rowsNum > 0 && !F.isEmpty(colNames);
+
+ List<List<?>> res = new ArrayList<>(rowsNum);
+
+ GridH2RowDescriptor desc = tbl.rowDescriptor();
+
+ for (List<DmlArgument> row : rows) {
+ List<Object> resRow = new ArrayList<>();
+
+ for (int j = 0; j < colNames.length; j++) {
+ Object colVal = row.get(j).get(args);
+
+ if (j == keyColIdx || j == valColIdx) {
+ Class<?> colCls = j == keyColIdx ? desc.type().keyClass() : desc.type().valueClass();
+
+ colVal = DmlUtils.convert(colVal, desc, colCls, colTypes[j]);
+ }
+
+ resRow.add(colVal);
+ }
+
+ res.add(resRow);
+ }
+
+ return res;
+ }
+
+ /**
* @return Update mode.
*/
public UpdateMode mode() {
@@ -379,11 +442,4 @@ public final class UpdatePlan {
@Nullable public boolean isLocalSubquery() {
return isLocSubqry;
}
-
- /**
- * @return Fast update.
- */
- @Nullable public FastUpdate fastUpdate() {
- return fastUpdate;
- }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ec290234/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
index a551639..5ffd264 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
@@ -21,6 +21,7 @@ import java.lang.reflect.Constructor;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -127,6 +128,8 @@ public final class UpdatePlanBuilder {
GridH2RowDescriptor desc;
+ List<GridSqlElement[]> elRows = null;
+
if (stmt instanceof GridSqlInsert) {
GridSqlInsert ins = (GridSqlInsert) stmt;
target = ins.into();
@@ -136,6 +139,10 @@ public final class UpdatePlanBuilder {
cols = ins.columns();
sel = DmlAstUtils.selectForInsertOrMerge(cols, ins.rows(), ins.query());
+
+ if (sel == null)
+ elRows = ins.rows();
+
isTwoStepSubqry = (ins.query() != null);
rowsNum = isTwoStepSubqry ? 0 : ins.rows().size();
}
@@ -149,6 +156,10 @@ public final class UpdatePlanBuilder {
cols = merge.columns();
sel = DmlAstUtils.selectForInsertOrMerge(cols, merge.rows(), merge.query());
+
+ if (sel == null)
+ elRows = merge.rows();
+
isTwoStepSubqry = (merge.query() != null);
rowsNum = isTwoStepSubqry ? 0 : merge.rows().size();
}
@@ -158,7 +169,7 @@ public final class UpdatePlanBuilder {
}
// Let's set the flag only for subqueries that have their FROM specified.
- isTwoStepSubqry = (isTwoStepSubqry && (sel instanceof GridSqlUnion ||
+ isTwoStepSubqry &= (sel != null && (sel instanceof GridSqlUnion ||
(sel instanceof GridSqlSelect && ((GridSqlSelect) sel).from() != null)));
int keyColIdx = -1;
@@ -210,13 +221,33 @@ public final class UpdatePlanBuilder {
KeyValueSupplier keySupplier = createSupplier(cctx, desc.type(), keyColIdx, hasKeyProps, true, false);
KeyValueSupplier valSupplier = createSupplier(cctx, desc.type(), valColIdx, hasValProps, false, false);
- String selectSql = sel.getSQL();
+ String selectSql = sel != null ? sel.getSQL() : null;
DmlDistributedPlanInfo distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ?
checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()) : null;
UpdateMode mode = stmt instanceof GridSqlMerge ? UpdateMode.MERGE : UpdateMode.INSERT;
+ List<List<DmlArgument>> rows = null;
+
+ if (elRows != null) {
+ assert sel == null;
+
+ rows = new ArrayList<>(elRows.size());
+
+ for (GridSqlElement[] elRow : elRows) {
+ List<DmlArgument> row = new ArrayList<>(cols.length);
+
+ for (GridSqlElement el : elRow) {
+ DmlArgument arg = DmlArguments.create(el);
+
+ row.add(arg);
+ }
+
+ rows.add(row);
+ }
+ }
+
return new UpdatePlan(
mode,
tbl.dataTable(),
@@ -228,6 +259,7 @@ public final class UpdatePlanBuilder {
valColIdx,
selectSql,
!isTwoStepSubqry,
+ rows,
rowsNum,
null,
distributed
@@ -349,6 +381,7 @@ public final class UpdatePlanBuilder {
valColIdx,
selectSql,
false,
+ null,
0,
null,
distributed
[39/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1fb53f38
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1fb53f38
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1fb53f38
Branch: refs/heads/ignite-zk
Commit: 1fb53f38d4ca5fbd42f9e036be2ddeecc4b38db6
Parents: 9e4f2c20c
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 18 18:15:12 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 18 18:15:12 2017 +0300
----------------------------------------------------------------------
.../ZookeeperDiscoverySpiBasicTest.java | 58 ++++++++++----------
1 file changed, 30 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1fb53f38/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index a29f478..58098b9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -1924,14 +1924,14 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testCommunicationErrorResolve_KillNode_1() throws Exception {
- communicationErrorResolve_KillNodes(2, Collections.singletonList(2L));
+ communicationErrorResolve_KillNodes(2, Collections.singleton(2L));
}
/**
* @throws Exception If failed.
*/
public void testCommunicationErrorResolve_KillNode_2() throws Exception {
- communicationErrorResolve_KillNodes(3, Collections.singletonList(2L));
+ communicationErrorResolve_KillNodes(3, Collections.singleton(2L));
}
/**
@@ -1942,6 +1942,34 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_1() throws Exception {
+ communicationErrorResolve_KillNodes(2, Collections.singleton(1L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_2() throws Exception {
+ communicationErrorResolve_KillNodes(3, Collections.singleton(1L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_3() throws Exception {
+ communicationErrorResolve_KillNodes(10, Arrays.asList(1L, 4L, 6L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator_4() throws Exception {
+ communicationErrorResolve_KillNodes(10, Arrays.asList(1L, 2L, 3L));
+ }
+
+ /**
* @param startNodes Number of nodes to start.
* @param killNodes Nodes to kill by resolve process.
* @throws Exception If failed.
@@ -1988,32 +2016,6 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
- public void testCommunicationErrorResolve_KillCoordinator() throws Exception {
- // Kill coordinator.
- testCommSpi = true;
-
- commProblemRslvr = new TestNodeKillCommunicationProblemResolver(Collections.singleton(1L));
-
- startGrids(3);
-
- ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(2));
-
- commSpi.checkRes = new BitSet(3);
-
- ZookeeperDiscoverySpi spi = spi(ignite(1));
-
- spi.onCommunicationConnectionError(ignite(0).cluster().localNode(), new Exception("test"));
-
- waitForTopology(2);
-
- startGrid(10);
-
- waitForTopology(3);
- }
-
- /**
- * @throws Exception If failed.
- */
public void testConnectionCheck() throws Exception {
final int NODES = 5;
[16/50] [abbrv] ignite git commit: IGNITE-7199 Visor CMD: minor
refactoring.
Posted by sb...@apache.org.
IGNITE-7199 Visor CMD: minor refactoring.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/cd195646
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/cd195646
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/cd195646
Branch: refs/heads/ignite-zk
Commit: cd1956461a0f310c9c6eaf600553015d4d7571f7
Parents: e08d19c
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Dec 14 12:08:58 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Dec 14 12:08:58 2017 +0700
----------------------------------------------------------------------
.../commands/tasks/VisorTasksCommand.scala | 34 +++++++++-----------
.../scala/org/apache/ignite/visor/visor.scala | 1 -
2 files changed, 15 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/cd195646/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala
----------------------------------------------------------------------
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala
index 0d6753e..966bd64 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala
@@ -17,23 +17,21 @@
package org.apache.ignite.visor.commands.tasks
+import java.util.UUID
+
import org.apache.ignite._
import org.apache.ignite.events.EventType._
import org.apache.ignite.internal.util.scala.impl
import org.apache.ignite.internal.util.typedef.X
import org.apache.ignite.internal.util.{IgniteUtils => U}
+import org.apache.ignite.internal.visor.event.{VisorGridEvent, VisorGridJobEvent, VisorGridTaskEvent}
+import org.apache.ignite.internal.visor.node.{VisorNodeEventsCollectorTask, VisorNodeEventsCollectorTaskArg}
+import org.apache.ignite.internal.visor.util.{VisorTaskUtils => TU}
import org.apache.ignite.lang.IgniteUuid
import org.apache.ignite.visor.VisorTag
import org.apache.ignite.visor.commands.common.{VisorConsoleCommand, VisorTextTable}
import org.apache.ignite.visor.visor._
-import java.util.UUID
-
-import org.apache.ignite.internal.visor.event.{VisorGridEvent, VisorGridJobEvent, VisorGridTaskEvent}
-import org.apache.ignite.internal.visor.node.VisorNodeEventsCollectorTask
-import org.apache.ignite.internal.visor.node.VisorNodeEventsCollectorTaskArg
-import org.apache.ignite.internal.visor.util.VisorTaskUtils._
-
import scala.collection.JavaConversions._
import scala.language.implicitConversions
import scala.util.control.Breaks._
@@ -1217,15 +1215,13 @@ class VisorTasksCommand extends VisorConsoleCommand {
eLst.foreach(e => {
e.nodeIds.foreach(id => {
- val host = sortAddresses(ignite.cluster.node(id).addresses).headOption
-
- if (host.isDefined) {
- var eSet = hMap.getOrElse(host.get, Set.empty[VisorExecution])
+ TU.sortAddresses(ignite.cluster.node(id).addresses).headOption.foreach(host => {
+ var eSet = hMap.getOrElse(host, Set.empty[VisorExecution])
eSet += e
- hMap += (host.get -> eSet)
- }
+ hMap += (host -> eSet)
+ })
})
})
@@ -1239,11 +1235,11 @@ class VisorTasksCommand extends VisorConsoleCommand {
tasksT.maxCellWidth = 55
- tasksT #=(
+ tasksT #= (
"Task Name(@), Oldest/Latest & Rate",
"Duration",
"Executions"
- )
+ )
println("Tasks executed on host " + host + ":")
@@ -1266,7 +1262,7 @@ class VisorTasksCommand extends VisorConsoleCommand {
val n = t.execs.size
- tasksT +=(
+ tasksT += (
(
t.taskNameVar,
" ",
@@ -1274,12 +1270,12 @@ class VisorTasksCommand extends VisorConsoleCommand {
"Latest: " + formatDateTime(t.latest),
" ",
"Exec. Rate: " + n + " in " + X.timeSpan2HMSM(t.timeframe)
- ),
+ ),
(
"min: " + X.timeSpan2HMSM(t.minDuration),
"avg: " + X.timeSpan2HMSM(t.avgDuration),
"max: " + X.timeSpan2HMSM(t.maxDuration)
- ),
+ ),
(
"Total: " + n,
" ",
@@ -1288,8 +1284,8 @@ class VisorTasksCommand extends VisorConsoleCommand {
"Fa: " + eE + " (" + formatInt(100 * eE / n) + "%)",
"Un: " + uE + " (" + formatInt(100 * uE / n) + "%)",
"Ti: " + tE + " (" + formatInt(100 * tE / n) + "%)"
- )
)
+ )
})
tasksT.render()
http://git-wip-us.apache.org/repos/asf/ignite/blob/cd195646/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
----------------------------------------------------------------------
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
index 41a14d0..5765579 100644
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala
@@ -35,7 +35,6 @@ import jline.console.ConsoleReader
import org.jetbrains.annotations.Nullable
import java.io._
import java.lang.{Boolean => JavaBoolean}
-import java.net._
import java.text._
import java.util.concurrent._
import java.util.{Collection => JavaCollection, HashSet => JavaHashSet, _}
[06/50] [abbrv] ignite git commit: IGNITE-7172 Fix minor CSS issues.
Posted by sb...@apache.org.
IGNITE-7172 Fix minor CSS issues.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/24412f52
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/24412f52
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/24412f52
Branch: refs/heads/ignite-zk
Commit: 24412f52f532c6e22c11f41c18d747400ac6891f
Parents: e6489ea
Author: Ilya Borisov <kl...@gmail.com>
Authored: Wed Dec 13 11:19:10 2017 +0700
Committer: Andrey Novikov <an...@gridgain.com>
Committed: Wed Dec 13 14:58:26 2017 +0700
----------------------------------------------------------------------
.../frontend/app/components/page-queries/style.scss | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/24412f52/modules/web-console/frontend/app/components/page-queries/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/style.scss b/modules/web-console/frontend/app/components/page-queries/style.scss
index 1f315e8..a937abf 100644
--- a/modules/web-console/frontend/app/components/page-queries/style.scss
+++ b/modules/web-console/frontend/app/components/page-queries/style.scss
@@ -45,6 +45,7 @@ page-queries {
top: 18px;
left: 14px;
width: 14px;
+ color: inherit;
}
ul {
@@ -56,6 +57,7 @@ page-queries {
.group {
background: white;
+ border-style: solid;
.group-legend {
label {
@@ -63,13 +65,15 @@ page-queries {
font-size: 12px;
color: $gray-light;
- background: none;
- padding-left: 0;
+ background: #fcfcfc;
+ padding: 0;
+ vertical-align: 1px;
}
}
.group-content {
height: 66px;
+ border-radius: 5px;
margin: 0;
padding: 5px 0;
[17/50] [abbrv] ignite git commit: IGNITE-7175 Throttling is not
applied to page allocation - Fixes #3209.
Posted by sb...@apache.org.
IGNITE-7175 Throttling is not applied to page allocation - Fixes #3209.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9ca3c096
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9ca3c096
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9ca3c096
Branch: refs/heads/ignite-zk
Commit: 9ca3c0964273920dcf3b122cff09a60614e9f6e7
Parents: cd19564
Author: dpavlov <dp...@gridgain.com>
Authored: Thu Dec 14 11:59:58 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Dec 14 11:59:58 2017 +0300
----------------------------------------------------------------------
.../processors/cache/persistence/pagemem/PageMemoryImpl.java | 2 +-
.../processors/cache/persistence/pagemem/PagesWriteThrottle.java | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9ca3c096/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 41de7f0..601f848 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1296,7 +1296,7 @@ public class PageMemoryImpl implements PageMemoryEx {
try {
rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
- if (throttleEnabled && !restore && markDirty && !dirty)
+ if (throttleEnabled && !restore && markDirty)
writeThrottle.onMarkDirty(isInCheckpoint(fullId));
}
catch (AssertionError ex) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9ca3c096/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
index d0c67c7..a890442 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
@@ -49,7 +49,8 @@ public class PagesWriteThrottle {
}
/**
- *
+ * Callback to apply throttling delay.
+ * @param isInCheckpoint flag indicating if checkpoint is running.
*/
public void onMarkDirty(boolean isInCheckpoint) {
assert dbSharedMgr.checkpointLockIsHeldByThread();
[20/50] [abbrv] ignite git commit: IGNITE-7200: SQL: simplified DML
module structure and restored encapsulation. This closes #3225.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
new file mode 100644
index 0000000..e662245
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdate.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.dml;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.query.h2.UpdateResult;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlConst;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement;
+import org.apache.ignite.internal.processors.query.h2.sql.GridSqlParameter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Arguments for fast, query-less UPDATE or DELETE - key and, optionally, value and new value.
+ */
+public final class FastUpdate {
+ /** Operand that always evaluates as {@code null}. */
+ private final static FastUpdateArgument NULL_ARG = new ConstantArgument(null);
+
+ /** Operand to compute key. */
+ private final FastUpdateArgument keyArg;
+
+ /** Operand to compute value. */
+ private final FastUpdateArgument valArg;
+
+ /** Operand to compute new value. */
+ private final FastUpdateArgument newValArg;
+
+ /**
+ * Create fast update instance.
+ *
+ * @param key Key element.
+ * @param val Value element.
+ * @param newVal New value element (if any)
+ * @return Fast update.
+ */
+ public static FastUpdate create(GridSqlElement key, GridSqlElement val, @Nullable GridSqlElement newVal) {
+ FastUpdateArgument keyArg = argument(key);
+ FastUpdateArgument valArg = argument(val);
+ FastUpdateArgument newValArg = argument(newVal);
+
+ return new FastUpdate(keyArg, valArg, newValArg);
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param keyArg Key argument.
+ * @param valArg Value argument.
+ * @param newValArg New value argument.
+ */
+ private FastUpdate(FastUpdateArgument keyArg, FastUpdateArgument valArg, FastUpdateArgument newValArg) {
+ this.keyArg = keyArg;
+ this.valArg = valArg;
+ this.newValArg = newValArg;
+ }
+
+ /**
+ * Perform single cache operation based on given args.
+ *
+ * @param cache Cache.
+ * @param args Query parameters.
+ * @return 1 if an item was affected, 0 otherwise.
+ * @throws IgniteCheckedException if failed.
+ */
+ @SuppressWarnings({"unchecked", "ConstantConditions"})
+ public UpdateResult execute(GridCacheAdapter cache, Object[] args) throws IgniteCheckedException {
+ Object key = keyArg.apply(args);
+
+ assert key != null;
+
+ Object val = valArg.apply(args);
+ Object newVal = newValArg.apply(args);
+
+ boolean res;
+
+ if (newVal != null) {
+ // Update.
+ if (val != null)
+ res = cache.replace(key, val, newVal);
+ else
+ res = cache.replace(key, newVal);
+ }
+ else {
+ // Delete.
+ if (val != null)
+ res = cache.remove(key, val);
+ else
+ res = cache.remove(key);
+ }
+
+ return res ? UpdateResult.ONE : UpdateResult.ZERO;
+ }
+
+ /**
+ * Create argument for AST element.
+ *
+ * @param el Element.
+ * @return Argument.
+ */
+ private static FastUpdateArgument argument(@Nullable GridSqlElement el) {
+ assert el == null ^ (el instanceof GridSqlConst || el instanceof GridSqlParameter);
+
+ if (el == null)
+ return NULL_ARG;
+
+ if (el instanceof GridSqlConst)
+ return new ConstantArgument(((GridSqlConst)el).value().getObject());
+ else
+ return new ParamArgument(((GridSqlParameter)el).index());
+ }
+
+ /**
+ * Value argument.
+ */
+ private static class ConstantArgument implements FastUpdateArgument {
+ /** Value to return. */
+ private final Object val;
+
+ /**
+ * Constructor.
+ *
+ * @param val Value.
+ */
+ private ConstantArgument(Object val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object apply(Object[] arg) throws IgniteCheckedException {
+ return val;
+ }
+ }
+
+ /**
+ * Parameter argument.
+ */
+ private static class ParamArgument implements FastUpdateArgument {
+ /** Value to return. */
+ private final int paramIdx;
+
+ /**
+ * Constructor.
+ *
+ * @param paramIdx Parameter index.
+ */
+ private ParamArgument(int paramIdx) {
+ assert paramIdx >= 0;
+
+ this.paramIdx = paramIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Object apply(Object[] arg) throws IgniteCheckedException {
+ assert arg.length > paramIdx;
+
+ return arg[paramIdx];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArguments.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArguments.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArguments.java
deleted file mode 100644
index cb47704..0000000
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/FastUpdateArguments.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.query.h2.dml;
-
-import org.apache.ignite.IgniteCheckedException;
-
-/**
- * Arguments for fast, query-less UPDATE or DELETE - key and, optionally, value and new value.
- */
-public final class FastUpdateArguments {
- /** Operand to compute key. */
- public final FastUpdateArgument key;
-
- /** Operand to compute value. */
- public final FastUpdateArgument val;
-
- /** Operand to compute new value. */
- public final FastUpdateArgument newVal;
-
- /** */
- public FastUpdateArguments(FastUpdateArgument key, FastUpdateArgument val, FastUpdateArgument newVal) {
- assert key != null && key != NULL_ARGUMENT;
- assert val != null;
- assert newVal != null;
-
- this.key = key;
- this.val = val;
- this.newVal = newVal;
- }
-
- /** Operand that always evaluates as {@code null}. */
- public final static FastUpdateArgument NULL_ARGUMENT = new FastUpdateArgument() {
- /** {@inheritDoc} */
- @Override public Object apply(Object[] arg) throws IgniteCheckedException {
- return null;
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
index a99d811..31dc52d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlan.java
@@ -17,64 +17,108 @@
package org.apache.ignite.internal.processors.query.h2.dml;
-import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.query.GridQueryProperty;
+import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
+import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.h2.table.Column;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap.DEFAULT_COLUMNS_COUNT;
/**
* Update plan - where to take data to update cache from and how to construct new keys and values, if needed.
*/
public final class UpdatePlan {
/** Initial statement to drive the rest of the logic. */
- public final UpdateMode mode;
+ private final UpdateMode mode;
/** Target table to be affected by initial DML statement. */
- public final GridH2Table tbl;
+ private final GridH2Table tbl;
/** Column names to set or update. */
- public final String[] colNames;
+ private final String[] colNames;
- /**
- * Expected column types to set or insert/merge.
- * @see org.h2.value.Value
- */
- public final int[] colTypes;
+ /** Column types to set for insert/merge. */
+ private final int[] colTypes;
/** Method to create key for INSERT or MERGE, ignored for UPDATE and DELETE. */
- public final KeyValueSupplier keySupplier;
+ private final KeyValueSupplier keySupplier;
/** Method to create value to put to cache, ignored for DELETE. */
- public final KeyValueSupplier valSupplier;
+ private final KeyValueSupplier valSupplier;
- /** Index of key column, if it's explicitly mentioned in column list of MERGE or INSERT,
- * ignored for UPDATE and DELETE. */
- public final int keyColIdx;
+ /** Key column index. */
+ private final int keyColIdx;
- /** Index of value column, if it's explicitly mentioned in column list. Ignored for UPDATE and DELETE. */
- public final int valColIdx;
+ /** Value column index. */
+ private final int valColIdx;
/** SELECT statement built upon initial DML statement. */
- public final String selectQry;
+ private final String selectQry;
/** Subquery flag - {@code true} if {@link #selectQry} is an actual subquery that retrieves data from some cache. */
- public final boolean isLocSubqry;
+ private final boolean isLocSubqry;
/** Number of rows in rows based MERGE or INSERT. */
- public final int rowsNum;
+ private final int rowsNum;
/** Arguments for fast UPDATE or DELETE. */
- public final FastUpdateArguments fastUpdateArgs;
+ private final FastUpdate fastUpdate;
/** Additional info for distributed update. */
- public final DistributedPlanInfo distributed;
+ private final DmlDistributedPlanInfo distributed;
- /** */
- private UpdatePlan(UpdateMode mode, GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier,
- KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry,
- int rowsNum, FastUpdateArguments fastUpdateArgs, DistributedPlanInfo distributed) {
+ /**
+ * Constructor.
+ *
+ * @param mode Mode.
+ * @param tbl Table.
+ * @param colNames Column names.
+ * @param colTypes Column types.
+ * @param keySupplier Key supplier.
+ * @param valSupplier Value supplier.
+ * @param keyColIdx Key column index.
+ * @param valColIdx value column index.
+ * @param selectQry Select query.
+ * @param isLocSubqry Local subquery flag.
+ * @param rowsNum Rows number.
+ * @param fastUpdate Fast update (if any).
+ * @param distributed Distributed plan (if any)
+ */
+ public UpdatePlan(
+ UpdateMode mode,
+ GridH2Table tbl,
+ String[] colNames,
+ int[] colTypes,
+ KeyValueSupplier keySupplier,
+ KeyValueSupplier valSupplier,
+ int keyColIdx,
+ int valColIdx,
+ String selectQry,
+ boolean isLocSubqry,
+ int rowsNum,
+ @Nullable FastUpdate fastUpdate,
+ @Nullable DmlDistributedPlanInfo distributed
+ ) {
this.colNames = colNames;
this.colTypes = colTypes;
this.rowsNum = rowsNum;
+
assert mode != null;
assert tbl != null;
@@ -86,85 +130,260 @@ public final class UpdatePlan {
this.valColIdx = valColIdx;
this.selectQry = selectQry;
this.isLocSubqry = isLocSubqry;
- this.fastUpdateArgs = fastUpdateArgs;
+ this.fastUpdate = fastUpdate;
this.distributed = distributed;
}
- /** */
- public static UpdatePlan forMerge(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier,
- KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry,
- int rowsNum, DistributedPlanInfo distributed) {
- assert !F.isEmpty(colNames);
-
- return new UpdatePlan(UpdateMode.MERGE, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx, valColIdx,
- selectQry, isLocSubqry, rowsNum, null, distributed);
+ /**
+ * Constructor for delete operation or fast update.
+ *
+ * @param mode Mode.
+ * @param tbl Table.
+ * @param selectQry Select query.
+ * @param fastUpdate Fast update arguments (if any).
+ * @param distributed Distributed plan (if any)
+ */
+ public UpdatePlan(
+ UpdateMode mode,
+ GridH2Table tbl,
+ String selectQry,
+ @Nullable FastUpdate fastUpdate,
+ @Nullable DmlDistributedPlanInfo distributed
+ ) {
+ this(
+ mode,
+ tbl,
+ null,
+ null,
+ null,
+ null,
+ -1,
+ -1,
+ selectQry,
+ false,
+ 0,
+ fastUpdate,
+ distributed
+ );
}
- /** */
- public static UpdatePlan forInsert(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier keySupplier,
- KeyValueSupplier valSupplier, int keyColIdx, int valColIdx, String selectQry, boolean isLocSubqry,
- int rowsNum, DistributedPlanInfo distributed) {
- assert !F.isEmpty(colNames);
+ /**
+ * Convert a row into key-value pair.
+ *
+ * @param row Row to process.
+ * @throws IgniteCheckedException if failed.
+ */
+ public IgniteBiTuple<?, ?> processRow(List<?> row) throws IgniteCheckedException {
+ GridH2RowDescriptor rowDesc = tbl.rowDescriptor();
+ GridQueryTypeDescriptor desc = rowDesc.type();
- return new UpdatePlan(UpdateMode.INSERT, tbl, colNames, colTypes, keySupplier, valSupplier, keyColIdx,
- valColIdx, selectQry, isLocSubqry, rowsNum, null, distributed);
- }
+ GridCacheContext cctx = rowDesc.context();
- /** */
- public static UpdatePlan forUpdate(GridH2Table tbl, String[] colNames, int[] colTypes, KeyValueSupplier valSupplier,
- int valColIdx, String selectQry, DistributedPlanInfo distributed) {
- assert !F.isEmpty(colNames);
+ Object key = keySupplier.apply(row);
- return new UpdatePlan(UpdateMode.UPDATE, tbl, colNames, colTypes, null, valSupplier, -1, valColIdx, selectQry,
- false, 0, null, distributed);
- }
+ if (QueryUtils.isSqlType(desc.keyClass())) {
+ assert keyColIdx != -1;
- /** */
- public static UpdatePlan forDelete(GridH2Table tbl, String selectQry, DistributedPlanInfo distributed) {
- return new UpdatePlan(UpdateMode.DELETE, tbl, null, null, null, null, -1, -1, selectQry, false, 0, null,
- distributed);
- }
+ key = DmlUtils.convert(key, rowDesc, desc.keyClass(), colTypes[keyColIdx]);
+ }
+
+ Object val = valSupplier.apply(row);
+
+ if (QueryUtils.isSqlType(desc.valueClass())) {
+ assert valColIdx != -1;
+
+ val = DmlUtils.convert(val, rowDesc, desc.valueClass(), colTypes[valColIdx]);
+ }
+
+ if (key == null) {
+ if (F.isEmpty(desc.keyFieldName()))
+ throw new IgniteSQLException("Key for INSERT or MERGE must not be null", IgniteQueryErrorCode.NULL_KEY);
+ else
+ throw new IgniteSQLException("Null value is not allowed for column '" + desc.keyFieldName() + "'",
+ IgniteQueryErrorCode.NULL_KEY);
+ }
- /** */
- public static UpdatePlan forFastUpdate(UpdateMode mode, GridH2Table tbl, FastUpdateArguments fastUpdateArgs) {
- assert mode == UpdateMode.UPDATE || mode == UpdateMode.DELETE;
+ if (val == null) {
+ if (F.isEmpty(desc.valueFieldName()))
+ throw new IgniteSQLException("Value for INSERT, MERGE, or UPDATE must not be null",
+ IgniteQueryErrorCode.NULL_VALUE);
+ else
+ throw new IgniteSQLException("Null value is not allowed for column '" + desc.valueFieldName() + "'",
+ IgniteQueryErrorCode.NULL_VALUE);
+ }
+
+ Map<String, Object> newColVals = new HashMap<>();
+
+ for (int i = 0; i < colNames.length; i++) {
+ if (i == keyColIdx || i == valColIdx)
+ continue;
+
+ String colName = colNames[i];
+
+ GridQueryProperty prop = desc.property(colName);
+
+ assert prop != null;
+
+ Class<?> expCls = prop.type();
+
+ newColVals.put(colName, DmlUtils.convert(row.get(i), rowDesc, expCls, colTypes[i]));
+ }
+
+ // We update columns in the order specified by the table for a reason - table's
+ // column order preserves their precedence for correct update of nested properties.
+ Column[] cols = tbl.getColumns();
+
+ // First 3 columns are _key, _val and _ver. Skip 'em.
+ for (int i = DEFAULT_COLUMNS_COUNT; i < cols.length; i++) {
+ if (tbl.rowDescriptor().isKeyValueOrVersionColumn(i))
+ continue;
+
+ String colName = cols[i].getName();
- return new UpdatePlan(mode, tbl, null, null, null, null, -1, -1, null, false, 0, fastUpdateArgs, null);
+ if (!newColVals.containsKey(colName))
+ continue;
+
+ Object colVal = newColVals.get(colName);
+
+ desc.setValue(colName, key, val, colVal);
+ }
+
+ if (cctx.binaryMarshaller()) {
+ if (key instanceof BinaryObjectBuilder)
+ key = ((BinaryObjectBuilder) key).build();
+
+ if (val instanceof BinaryObjectBuilder)
+ val = ((BinaryObjectBuilder) val).build();
+ }
+
+ desc.validateKeyAndValue(key, val);
+
+ return new IgniteBiTuple<>(key, val);
}
/**
- * Additional information about distributed update plan.
+ * Convert a row into value.
+ *
+ * @param row Row to process.
+ * @throws IgniteCheckedException if failed.
*/
- public final static class DistributedPlanInfo {
- /** Whether update involves only replicated caches. */
- private final boolean replicatedOnly;
-
- /** Identifiers of caches involved in update (used for cluster nodes mapping). */
- private final List<Integer> cacheIds;
-
- /**
- * Constructor.
- *
- * @param replicatedOnly Whether all caches are replicated.
- * @param cacheIds List of cache identifiers.
- */
- DistributedPlanInfo(boolean replicatedOnly, List<Integer> cacheIds) {
- this.replicatedOnly = replicatedOnly;
- this.cacheIds = cacheIds;
+ public T3<Object, Object, Object> processRowForUpdate(List<?> row) throws IgniteCheckedException {
+ GridH2RowDescriptor rowDesc = tbl.rowDescriptor();
+ GridQueryTypeDescriptor desc = rowDesc.type();
+
+ GridCacheContext cctx = rowDesc.context();
+
+ boolean hasNewVal = (valColIdx != -1);
+
+ boolean hasProps = !hasNewVal || colNames.length > 1;
+
+ Object key = row.get(0);
+
+ Object oldVal = row.get(1);
+
+ if (cctx.binaryMarshaller() && !(oldVal instanceof BinaryObject))
+ oldVal = cctx.grid().binary().toBinary(oldVal);
+
+ Object newVal;
+
+ Map<String, Object> newColVals = new HashMap<>();
+
+ for (int i = 0; i < colNames.length; i++) {
+ if (hasNewVal && i == valColIdx - 2)
+ continue;
+
+ GridQueryProperty prop = tbl.rowDescriptor().type().property(colNames[i]);
+
+ assert prop != null : "Unknown property: " + colNames[i];
+
+ newColVals.put(colNames[i], DmlUtils.convert(row.get(i + 2), rowDesc, prop.type(), colTypes[i]));
}
- /**
- * @return {@code true} in case all involved caches are replicated.
- */
- public boolean isReplicatedOnly() {
- return replicatedOnly;
+ newVal = valSupplier.apply(row);
+
+ if (newVal == null)
+ throw new IgniteSQLException("New value for UPDATE must not be null", IgniteQueryErrorCode.NULL_VALUE);
+
+ // Skip key and value - that's why we start off with 3rd column
+ for (int i = 0; i < tbl.getColumns().length - DEFAULT_COLUMNS_COUNT; i++) {
+ Column c = tbl.getColumn(i + DEFAULT_COLUMNS_COUNT);
+
+ if (rowDesc.isKeyValueOrVersionColumn(c.getColumnId()))
+ continue;
+
+ GridQueryProperty prop = desc.property(c.getName());
+
+ if (prop.key())
+ continue; // Don't get values of key's columns - we won't use them anyway
+
+ boolean hasNewColVal = newColVals.containsKey(c.getName());
+
+ if (!hasNewColVal)
+ continue;
+
+ Object colVal = newColVals.get(c.getName());
+
+ // UPDATE currently does not allow to modify key or its fields, so we must be safe to pass null as key.
+ rowDesc.setColumnValue(null, newVal, colVal, i);
}
- /**
- * @return cache identifiers.
- */
- public List<Integer> getCacheIds() {
- return cacheIds;
+ if (cctx.binaryMarshaller() && hasProps) {
+ assert newVal instanceof BinaryObjectBuilder;
+
+ newVal = ((BinaryObjectBuilder) newVal).build();
}
+
+ desc.validateKeyAndValue(key, newVal);
+
+ return new T3<>(key, oldVal, newVal);
+ }
+
+ /**
+ * @return Update mode.
+ */
+ public UpdateMode mode() {
+ return mode;
+ }
+
+ /**
+ * @return Cache context.
+ */
+ public GridCacheContext cacheContext() {
+ return tbl.cache();
+ }
+
+ /**
+ * @return Distributed plan info (for skip-reducer mode).
+ */
+ @Nullable public DmlDistributedPlanInfo distributedPlan() {
+ return distributed;
+ }
+
+ /**
+ * @return Row count.
+ */
+ public int rowCount() {
+ return rowsNum;
+ }
+
+ /**
+ * @return Select query.
+ */
+ public String selectQuery() {
+ return selectQry;
+ }
+
+ /**
+ * @return Local subquery flag.
+ */
+ @Nullable public boolean isLocalSubquery() {
+ return isLocSubqry;
+ }
+
+ /**
+ * @return Fast update.
+ */
+ @Nullable public FastUpdate fastUpdate() {
+ return fastUpdate;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
index c845266..a551639 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
@@ -40,7 +40,6 @@ import org.apache.ignite.internal.processors.query.h2.DmlStatementsProcessor;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
-import org.apache.ignite.internal.processors.query.h2.sql.DmlAstUtils;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlColumn;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlDelete;
import org.apache.ignite.internal.processors.query.h2.sql.GridSqlElement;
@@ -213,15 +212,26 @@ public final class UpdatePlanBuilder {
String selectSql = sel.getSQL();
- UpdatePlan.DistributedPlanInfo distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ?
+ DmlDistributedPlanInfo distributed = (rowsNum == 0 && !F.isEmpty(selectSql)) ?
checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName()) : null;
- if (stmt instanceof GridSqlMerge)
- return UpdatePlan.forMerge(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx,
- valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed);
- else
- return UpdatePlan.forInsert(tbl.dataTable(), colNames, colTypes, keySupplier, valSupplier, keyColIdx,
- valColIdx, selectSql, !isTwoStepSubqry, rowsNum, distributed);
+ UpdateMode mode = stmt instanceof GridSqlMerge ? UpdateMode.MERGE : UpdateMode.INSERT;
+
+ return new UpdatePlan(
+ mode,
+ tbl.dataTable(),
+ colNames,
+ colTypes,
+ keySupplier,
+ valSupplier,
+ keyColIdx,
+ valColIdx,
+ selectSql,
+ !isTwoStepSubqry,
+ rowsNum,
+ null,
+ distributed
+ );
}
/**
@@ -241,7 +251,7 @@ public final class UpdatePlanBuilder {
throws IgniteCheckedException {
GridSqlElement target;
- FastUpdateArguments fastUpdate;
+ FastUpdate fastUpdate;
UpdateMode mode;
@@ -249,7 +259,7 @@ public final class UpdatePlanBuilder {
// Let's verify that user is not trying to mess with key's columns directly
verifyUpdateColumns(stmt);
- GridSqlUpdate update = (GridSqlUpdate) stmt;
+ GridSqlUpdate update = (GridSqlUpdate)stmt;
target = update.target();
fastUpdate = DmlAstUtils.getFastUpdateArgs(update);
mode = UpdateMode.UPDATE;
@@ -266,16 +276,23 @@ public final class UpdatePlanBuilder {
GridSqlTable tbl = DmlAstUtils.gridTableForElement(target);
- GridH2Table gridTbl = tbl.dataTable();
+ GridH2Table h2Tbl = tbl.dataTable();
- GridH2RowDescriptor desc = gridTbl.rowDescriptor();
+ GridH2RowDescriptor desc = h2Tbl.rowDescriptor();
if (desc == null)
- throw new IgniteSQLException("Row descriptor undefined for table '" + gridTbl.getName() + "'",
+ throw new IgniteSQLException("Row descriptor undefined for table '" + h2Tbl.getName() + "'",
IgniteQueryErrorCode.NULL_TABLE_DESCRIPTOR);
- if (fastUpdate != null)
- return UpdatePlan.forFastUpdate(mode, gridTbl, fastUpdate);
+ if (fastUpdate != null) {
+ return new UpdatePlan(
+ mode,
+ h2Tbl,
+ null,
+ fastUpdate,
+ null
+ );
+ }
else {
GridSqlSelect sel;
@@ -311,28 +328,47 @@ public final class UpdatePlanBuilder {
int newValColIdx = (hasNewVal ? valColIdx : 1);
- KeyValueSupplier newValSupplier = createSupplier(desc.context(), desc.type(), newValColIdx, hasProps,
+ KeyValueSupplier valSupplier = createSupplier(desc.context(), desc.type(), newValColIdx, hasProps,
false, true);
sel = DmlAstUtils.selectForUpdate((GridSqlUpdate) stmt, errKeysPos);
String selectSql = sel.getSQL();
- UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null :
+ DmlDistributedPlanInfo distributed = F.isEmpty(selectSql) ? null :
checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName());
- return UpdatePlan.forUpdate(gridTbl, colNames, colTypes, newValSupplier, valColIdx, selectSql,
- distributed);
+ return new UpdatePlan(
+ UpdateMode.UPDATE,
+ h2Tbl,
+ colNames,
+ colTypes,
+ null,
+ valSupplier,
+ -1,
+ valColIdx,
+ selectSql,
+ false,
+ 0,
+ null,
+ distributed
+ );
}
else {
sel = DmlAstUtils.selectForDelete((GridSqlDelete) stmt, errKeysPos);
String selectSql = sel.getSQL();
- UpdatePlan.DistributedPlanInfo distributed = F.isEmpty(selectSql) ? null :
+ DmlDistributedPlanInfo distributed = F.isEmpty(selectSql) ? null :
checkPlanCanBeDistributed(idx, conn, fieldsQuery, loc, selectSql, tbl.dataTable().cacheName());
- return UpdatePlan.forDelete(gridTbl, selectSql, distributed);
+ return new UpdatePlan(
+ UpdateMode.DELETE,
+ h2Tbl,
+ selectSql,
+ null,
+ distributed
+ );
}
}
}
@@ -546,7 +582,7 @@ public final class UpdatePlanBuilder {
* @return distributed update plan info, or {@code null} if cannot be distributed.
* @throws IgniteCheckedException if failed.
*/
- private static UpdatePlan.DistributedPlanInfo checkPlanCanBeDistributed(IgniteH2Indexing idx,
+ private static DmlDistributedPlanInfo checkPlanCanBeDistributed(IgniteH2Indexing idx,
Connection conn, SqlFieldsQuery fieldsQry, boolean loc, String selectQry, String cacheName)
throws IgniteCheckedException {
@@ -570,7 +606,7 @@ public final class UpdatePlanBuilder {
boolean distributed = qry.skipMergeTable() && qry.mapQueries().size() == 1 &&
!qry.mapQueries().get(0).hasSubQueries();
- return distributed ? new UpdatePlan.DistributedPlanInfo(qry.isReplicatedOnly(),
+ return distributed ? new DmlDistributedPlanInfo(qry.isReplicatedOnly(),
idx.collectCacheIds(CU.cacheId(cacheName), qry)): null;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
deleted file mode 100644
index 4537ffc..0000000
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/DmlAstUtils.java
+++ /dev/null
@@ -1,644 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.query.h2.sql;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
-import org.apache.ignite.internal.processors.query.IgniteSQLException;
-import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArgument;
-import org.apache.ignite.internal.processors.query.h2.dml.FastUpdateArguments;
-import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap;
-import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
-import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
-import org.apache.ignite.internal.util.lang.IgnitePair;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.h2.command.Parser;
-import org.h2.expression.Expression;
-import org.h2.table.Column;
-import org.h2.table.Table;
-import org.h2.util.IntArray;
-import org.h2.value.DataType;
-import org.h2.value.Value;
-import org.h2.value.ValueDate;
-import org.h2.value.ValueInt;
-import org.h2.value.ValueString;
-import org.h2.value.ValueTime;
-import org.h2.value.ValueTimestamp;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * AST utils for DML
- */
-public final class DmlAstUtils {
- /**
- * Empty ctor to prevent initialization.
- */
- private DmlAstUtils() {
- // No-op.
- }
-
- /**
- * Create SELECT on which subsequent INSERT or MERGE will be based.
- *
- * @param cols Columns to insert values into.
- * @param rows Rows to create pseudo-SELECT upon.
- * @param subQry Subquery to use rather than rows.
- * @return Subquery or pseudo-SELECT to evaluate inserted expressions.
- */
- public static GridSqlQuery selectForInsertOrMerge(GridSqlColumn[] cols, List<GridSqlElement[]> rows,
- GridSqlQuery subQry) {
- if (!F.isEmpty(rows)) {
- assert !F.isEmpty(cols);
-
- GridSqlSelect sel = new GridSqlSelect();
-
- GridSqlFunction from = new GridSqlFunction(GridSqlFunctionType.TABLE);
-
- sel.from(from);
-
- GridSqlArray[] args = new GridSqlArray[cols.length];
-
- for (int i = 0; i < cols.length; i++) {
- GridSqlArray arr = new GridSqlArray(rows.size());
-
- String colName = cols[i].columnName();
-
- GridSqlAlias alias = new GridSqlAlias(colName, arr);
-
- alias.resultType(cols[i].resultType());
-
- from.addChild(alias);
-
- args[i] = arr;
-
- GridSqlColumn newCol = new GridSqlColumn(null, from, null,"TABLE", colName);
-
- newCol.resultType(cols[i].resultType());
-
- sel.addColumn(newCol, true);
- }
-
- for (GridSqlElement[] row : rows) {
- assert cols.length == row.length;
-
- for (int i = 0; i < row.length; i++)
- args[i].addChild(row[i]);
- }
-
- return sel;
- }
- else {
- assert subQry != null;
-
- return subQry;
- }
- }
-
- /**
- * Generate SQL SELECT based on DELETE's WHERE, LIMIT, etc.
- *
- * @param del Delete statement.
- * @param keysParamIdx Index for .
- * @return SELECT statement.
- */
- public static GridSqlSelect selectForDelete(GridSqlDelete del, @Nullable Integer keysParamIdx) {
- GridSqlSelect mapQry = new GridSqlSelect();
-
- mapQry.from(del.from());
-
- Set<GridSqlTable> tbls = new HashSet<>();
-
- collectAllGridTablesInTarget(del.from(), tbls);
-
- assert tbls.size() == 1 : "Failed to determine target table for DELETE";
-
- GridSqlTable tbl = tbls.iterator().next();
-
- GridH2Table gridTbl = tbl.dataTable();
-
- assert gridTbl != null : "Failed to determine target grid table for DELETE";
-
- Column h2KeyCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.KEY_COL);
-
- Column h2ValCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.VAL_COL);
-
- GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName());
- keyCol.resultType(GridSqlType.fromColumn(h2KeyCol));
-
- GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName());
- valCol.resultType(GridSqlType.fromColumn(h2ValCol));
-
- mapQry.addColumn(keyCol, true);
- mapQry.addColumn(valCol, true);
-
- GridSqlElement where = del.where();
- if (keysParamIdx != null)
- where = injectKeysFilterParam(where, keyCol, keysParamIdx);
-
- mapQry.where(where);
- mapQry.limit(del.limit());
-
- return mapQry;
- }
-
- /**
- * @param update UPDATE statement.
- * @return {@code null} if given statement directly updates {@code _val} column with a literal or param value
- * and filters by single non expression key (and, optionally, by single non expression value).
- */
- public static FastUpdateArguments getFastUpdateArgs(GridSqlUpdate update) {
- IgnitePair<GridSqlElement> filter = findKeyValueEqualityCondition(update.where());
-
- if (filter == null)
- return null;
-
- if (update.cols().size() != 1)
- return null;
-
- Table tbl = update.cols().get(0).column().getTable();
- if (!(tbl instanceof GridH2Table))
- return null;
-
- GridH2RowDescriptor desc = ((GridH2Table)tbl).rowDescriptor();
- if (!desc.isValueColumn(update.cols().get(0).column().getColumnId()))
- return null;
-
- GridSqlElement set = update.set().get(update.cols().get(0).columnName());
-
- if (!(set instanceof GridSqlConst || set instanceof GridSqlParameter))
- return null;
-
- return new FastUpdateArguments(operandForElement(filter.getKey()), operandForElement(filter.getValue()),
- operandForElement(set));
- }
-
- /**
- * Create operand based on exact type of SQL element.
- *
- * @param el element.
- * @return Operand.
- */
- private static FastUpdateArgument operandForElement(GridSqlElement el) {
- assert el == null ^ (el instanceof GridSqlConst || el instanceof GridSqlParameter);
-
- if (el == null)
- return FastUpdateArguments.NULL_ARGUMENT;
-
- if (el instanceof GridSqlConst)
- return new ValueArgument(((GridSqlConst)el).value().getObject());
- else
- return new ParamArgument(((GridSqlParameter)el).index());
- }
-
- /**
- * @param del DELETE statement.
- * @return {@code true} if given statement filters by single non expression key.
- */
- public static FastUpdateArguments getFastDeleteArgs(GridSqlDelete del) {
- IgnitePair<GridSqlElement> filter = findKeyValueEqualityCondition(del.where());
-
- if (filter == null)
- return null;
-
- return new FastUpdateArguments(operandForElement(filter.getKey()), operandForElement(filter.getValue()),
- FastUpdateArguments.NULL_ARGUMENT);
- }
-
- /**
- * @param where Element to test.
- * @return Whether given element corresponds to {@code WHERE _key = ?}, and key is a literal expressed
- * in query or a query param.
- */
- private static IgnitePair<GridSqlElement> findKeyValueEqualityCondition(GridSqlElement where) {
- if (where == null || !(where instanceof GridSqlOperation))
- return null;
-
- GridSqlOperation whereOp = (GridSqlOperation) where;
-
- // Does this WHERE limit only by _key?
- if (isKeyEqualityCondition(whereOp))
- return new IgnitePair<>((GridSqlElement)whereOp.child(1), null);
-
- // Or maybe it limits both by _key and _val?
- if (whereOp.operationType() != GridSqlOperationType.AND)
- return null;
-
- GridSqlElement left = whereOp.child(0);
-
- GridSqlElement right = whereOp.child(1);
-
- if (!(left instanceof GridSqlOperation && right instanceof GridSqlOperation))
- return null;
-
- GridSqlOperation leftOp = (GridSqlOperation) left;
-
- GridSqlOperation rightOp = (GridSqlOperation) right;
-
- if (isKeyEqualityCondition(leftOp)) { // _key = ? and _val = ?
- if (!isValueEqualityCondition(rightOp))
- return null;
-
- return new IgnitePair<>((GridSqlElement)leftOp.child(1), (GridSqlElement)rightOp.child(1));
- }
- else if (isKeyEqualityCondition(rightOp)) { // _val = ? and _key = ?
- if (!isValueEqualityCondition(leftOp))
- return null;
-
- return new IgnitePair<>((GridSqlElement)rightOp.child(1), (GridSqlElement)leftOp.child(1));
- }
- else // Neither
- return null;
- }
-
- /**
- * @param op Operation.
- * @param key true - check for key equality condition,
- * otherwise check for value equality condition
- * @return Whether this condition is of form {@code colName} = ?
- */
- private static boolean isEqualityCondition(GridSqlOperation op, boolean key) {
- if (op.operationType() != GridSqlOperationType.EQUAL)
- return false;
-
- GridSqlElement left = op.child(0);
- GridSqlElement right = op.child(1);
-
- if (!(left instanceof GridSqlColumn))
- return false;
-
- GridSqlColumn column = (GridSqlColumn)left;
- if (!(column.column().getTable() instanceof GridH2Table))
- return false;
-
- GridH2RowDescriptor desc =((GridH2Table) column.column().getTable()).rowDescriptor();
-
- return (key ? desc.isKeyColumn(column.column().getColumnId()) :
- desc.isValueColumn(column.column().getColumnId())) &&
- (right instanceof GridSqlConst || right instanceof GridSqlParameter);
- }
-
- /**
- * @param op Operation.
- * @return Whether this condition is of form _key = ?
- */
- private static boolean isKeyEqualityCondition(GridSqlOperation op) {
- return isEqualityCondition(op, true);
- }
-
- /**
- * @param op Operation.
- * @return Whether this condition is of form _val = ?
- */
- private static boolean isValueEqualityCondition(GridSqlOperation op) {
- return isEqualityCondition(op, false);
- }
-
-
- /**
- * Generate SQL SELECT based on UPDATE's WHERE, LIMIT, etc.
- *
- * @param update Update statement.
- * @param keysParamIdx Index of new param for the array of keys.
- * @return SELECT statement.
- */
- public static GridSqlSelect selectForUpdate(GridSqlUpdate update, @Nullable Integer keysParamIdx) {
- GridSqlSelect mapQry = new GridSqlSelect();
-
- mapQry.from(update.target());
-
- Set<GridSqlTable> tbls = new HashSet<>();
-
- collectAllGridTablesInTarget(update.target(), tbls);
-
- assert tbls.size() == 1 : "Failed to determine target table for UPDATE";
-
- GridSqlTable tbl = tbls.iterator().next();
-
- GridH2Table gridTbl = tbl.dataTable();
-
- assert gridTbl != null : "Failed to determine target grid table for UPDATE";
-
- Column h2KeyCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.KEY_COL);
-
- Column h2ValCol = gridTbl.getColumn(GridH2KeyValueRowOnheap.VAL_COL);
-
- GridSqlColumn keyCol = new GridSqlColumn(h2KeyCol, tbl, h2KeyCol.getName());
- keyCol.resultType(GridSqlType.fromColumn(h2KeyCol));
-
- GridSqlColumn valCol = new GridSqlColumn(h2ValCol, tbl, h2ValCol.getName());
- valCol.resultType(GridSqlType.fromColumn(h2ValCol));
-
- mapQry.addColumn(keyCol, true);
- mapQry.addColumn(valCol, true);
-
- for (GridSqlColumn c : update.cols()) {
- String newColName = Parser.quoteIdentifier("_upd_" + c.columnName());
- // We have to use aliases to cover cases when the user
- // wants to update _val field directly (if it's a literal)
- GridSqlAlias alias = new GridSqlAlias(newColName, elementOrDefault(update.set().get(c.columnName()), c), true);
- alias.resultType(c.resultType());
- mapQry.addColumn(alias, true);
- }
-
- GridSqlElement where = update.where();
- if (keysParamIdx != null)
- where = injectKeysFilterParam(where, keyCol, keysParamIdx);
-
- mapQry.where(where);
- mapQry.limit(update.limit());
-
- return mapQry;
- }
-
- /**
- * Do what we can to compute default value for this column (mimics H2 behavior).
- * @see Table#getDefaultValue
- * @see Column#validateConvertUpdateSequence
- * @param el SQL element.
- * @param col Column.
- * @return {@link GridSqlConst#NULL}, if {@code el} is null, or {@code el} if
- * it's not {@link GridSqlKeyword#DEFAULT}, or computed default value.
- */
- private static GridSqlElement elementOrDefault(GridSqlElement el, GridSqlColumn col) {
- if (el == null)
- return GridSqlConst.NULL;
-
- if (el != GridSqlKeyword.DEFAULT)
- return el;
-
- Column h2Col = col.column();
-
- Expression dfltExpr = h2Col.getDefaultExpression();
-
- Value dfltVal;
-
- try {
- dfltVal = dfltExpr != null ? dfltExpr.getValue(null) : null;
- }
- catch (Exception ignored) {
- throw new IgniteSQLException("Failed to evaluate default value for a column " + col.columnName());
- }
-
- if (dfltVal != null)
- return new GridSqlConst(dfltVal);
-
- int type = h2Col.getType();
-
- DataType dt = DataType.getDataType(type);
-
- if (dt.decimal)
- dfltVal = ValueInt.get(0).convertTo(type);
- else if (dt.type == Value.TIMESTAMP)
- dfltVal = ValueTimestamp.fromMillis(U.currentTimeMillis());
- else if (dt.type == Value.TIME)
- dfltVal = ValueTime.fromNanos(0);
- else if (dt.type == Value.DATE)
- dfltVal = ValueDate.fromMillis(U.currentTimeMillis());
- else
- dfltVal = ValueString.get("").convertTo(type);
-
- return new GridSqlConst(dfltVal);
- }
-
- /**
- * Append additional condition to WHERE for it to select only specific keys.
- *
- * @param where Initial condition.
- * @param keyCol Column to base the new condition on.
- * @return New condition.
- */
- private static GridSqlElement injectKeysFilterParam(GridSqlElement where, GridSqlColumn keyCol, int paramIdx) {
- // Yes, we need a subquery for "WHERE _key IN ?" to work with param being an array without dirty query rewriting.
- GridSqlSelect sel = new GridSqlSelect();
-
- GridSqlFunction from = new GridSqlFunction(GridSqlFunctionType.TABLE);
-
- sel.from(from);
-
- GridSqlColumn col = new GridSqlColumn(null, from, null, "TABLE", "_IGNITE_ERR_KEYS");
-
- sel.addColumn(col, true);
-
- GridSqlAlias alias = new GridSqlAlias("_IGNITE_ERR_KEYS", new GridSqlParameter(paramIdx));
-
- alias.resultType(keyCol.resultType());
-
- from.addChild(alias);
-
- GridSqlElement e = new GridSqlOperation(GridSqlOperationType.IN, keyCol, new GridSqlSubquery(sel));
-
- if (where == null)
- return e;
- else
- return new GridSqlOperation(GridSqlOperationType.AND, where, e);
- }
-
- /**
- * @param qry Select.
- * @param params Parameters.
- * @param target Extracted parameters.
- * @param paramIdxs Parameter indexes.
- * @return Extracted parameters list.
- */
- private static List<Object> findParams(GridSqlQuery qry, Object[] params, ArrayList<Object> target,
- IntArray paramIdxs) {
- if (qry instanceof GridSqlSelect)
- return findParams((GridSqlSelect)qry, params, target, paramIdxs);
-
- GridSqlUnion union = (GridSqlUnion)qry;
-
- findParams(union.left(), params, target, paramIdxs);
- findParams(union.right(), params, target, paramIdxs);
-
- findParams((GridSqlElement)qry.limit(), params, target, paramIdxs);
- findParams((GridSqlElement)qry.offset(), params, target, paramIdxs);
-
- return target;
- }
-
- /**
- * @param qry Select.
- * @param params Parameters.
- * @param target Extracted parameters.
- * @param paramIdxs Parameter indexes.
- * @return Extracted parameters list.
- */
- private static List<Object> findParams(GridSqlSelect qry, Object[] params, ArrayList<Object> target,
- IntArray paramIdxs) {
- if (params.length == 0)
- return target;
-
- for (GridSqlAst el : qry.columns(false))
- findParams((GridSqlElement)el, params, target, paramIdxs);
-
- findParams((GridSqlElement)qry.from(), params, target, paramIdxs);
- findParams((GridSqlElement)qry.where(), params, target, paramIdxs);
-
- // Don't search in GROUP BY and HAVING since they expected to be in select list.
-
- findParams((GridSqlElement)qry.limit(), params, target, paramIdxs);
- findParams((GridSqlElement)qry.offset(), params, target, paramIdxs);
-
- return target;
- }
-
- /**
- * @param el Element.
- * @param params Parameters.
- * @param target Extracted parameters.
- * @param paramIdxs Parameter indexes.
- */
- private static void findParams(@Nullable GridSqlElement el, Object[] params, ArrayList<Object> target,
- IntArray paramIdxs) {
- if (el == null)
- return;
-
- if (el instanceof GridSqlParameter) {
- // H2 Supports queries like "select ?5" but first 4 non-existing parameters are need to be set to any value.
- // Here we will set them to NULL.
- final int idx = ((GridSqlParameter)el).index();
-
- while (target.size() < idx)
- target.add(null);
-
- if (params.length <= idx)
- throw new IgniteException("Invalid number of query parameters. " +
- "Cannot find " + idx + " parameter.");
-
- Object param = params[idx];
-
- if (idx == target.size())
- target.add(param);
- else
- target.set(idx, param);
-
- paramIdxs.add(idx);
- }
- else if (el instanceof GridSqlSubquery)
- findParams(((GridSqlSubquery)el).subquery(), params, target, paramIdxs);
- else
- for (int i = 0; i < el.size(); i++)
- findParams((GridSqlElement)el.child(i), params, target, paramIdxs);
- }
-
- /**
- * Processes all the tables and subqueries using the given closure.
- *
- * @param from FROM element.
- * @param c Closure each found table and subquery will be passed to. If returns {@code true} the we need to stop.
- * @return {@code true} If we have found.
- */
- private static boolean findTablesInFrom(GridSqlElement from, IgnitePredicate<GridSqlElement> c) {
- if (from == null)
- return false;
-
- if (from instanceof GridSqlTable || from instanceof GridSqlSubquery)
- return c.apply(from);
-
- if (from instanceof GridSqlJoin) {
- // Left and right.
- if (findTablesInFrom((GridSqlElement)from.child(0), c))
- return true;
-
- if (findTablesInFrom((GridSqlElement)from.child(1), c))
- return true;
-
- // We don't process ON condition because it is not a joining part of from here.
- return false;
- }
- else if (from instanceof GridSqlAlias)
- return findTablesInFrom((GridSqlElement)from.child(), c);
- else if (from instanceof GridSqlFunction)
- return false;
-
- throw new IllegalStateException(from.getClass().getName() + " : " + from.getSQL());
- }
-
- /**
- * @param from From element.
- * @param tbls Tables.
- */
- public static void collectAllGridTablesInTarget(GridSqlElement from, final Set<GridSqlTable> tbls) {
- findTablesInFrom(from, new IgnitePredicate<GridSqlElement>() {
- @Override public boolean apply(GridSqlElement el) {
- if (el instanceof GridSqlTable)
- tbls.add((GridSqlTable)el);
-
- return false;
- }
- });
- }
-
- /**
- * @param target Expression to extract the table from.
- * @return Back end table for this element.
- */
- public static GridSqlTable gridTableForElement(GridSqlElement target) {
- Set<GridSqlTable> tbls = new HashSet<>();
-
- collectAllGridTablesInTarget(target, tbls);
-
- if (tbls.size() != 1)
- throw new IgniteSQLException("Failed to determine target table", IgniteQueryErrorCode.TABLE_NOT_FOUND);
-
- return tbls.iterator().next();
- }
-
- /** Simple constant value based operand. */
- private final static class ValueArgument implements FastUpdateArgument {
- /** Value to return. */
- private final Object val;
-
- /** */
- private ValueArgument(Object val) {
- this.val = val;
- }
-
- /** {@inheritDoc} */
- @Override public Object apply(Object[] arg) throws IgniteCheckedException {
- return val;
- }
- }
-
- /** Simple constant value based operand. */
- private final static class ParamArgument implements FastUpdateArgument {
- /** Value to return. */
- private final int paramIdx;
-
- /** */
- private ParamArgument(int paramIdx) {
- assert paramIdx >= 0;
-
- this.paramIdx = paramIdx;
- }
-
- /** {@inheritDoc} */
- @Override public Object apply(Object[] arg) throws IgniteCheckedException {
- assert arg.length > paramIdx;
-
- return arg[paramIdx];
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/03bb5513/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
index 8ffc5fa..3b19e39 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQueryParser.java
@@ -36,6 +36,7 @@ import org.apache.ignite.cache.QueryIndexType;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.h2.dml.DmlAstUtils;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
import org.apache.ignite.internal.util.typedef.F;
[24/50] [abbrv] ignite git commit: IGNITE-7208 Fixed pug template.
Posted by sb...@apache.org.
IGNITE-7208 Fixed pug template.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/061ec6ae
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/061ec6ae
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/061ec6ae
Branch: refs/heads/ignite-zk
Commit: 061ec6aef7ecc60ac57fa2295b5687560be1c29b
Parents: da78295
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Fri Dec 15 10:19:22 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Dec 15 10:19:22 2017 +0700
----------------------------------------------------------------------
.../frontend/app/components/page-queries/template.tpl.pug | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/061ec6ae/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
index 1574874..4419d5c 100644
--- a/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/template.tpl.pug
@@ -369,7 +369,7 @@ mixin paragraph-query
ignite-information(
data-title='With query notebook you can'
style='margin-bottom: 30px'
- ng-init=`example = '${example}'`
+ ng-init=`example = "${example}"`
)
ul
li Create any number of queries
[49/50] [abbrv] ignite git commit: Throttle new events generation
Posted by sb...@apache.org.
Throttle new events generation
(cherry picked from commit d29ea50)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f9698f4d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f9698f4d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f9698f4d
Branch: refs/heads/ignite-zk
Commit: f9698f4d60dad816e33c6d6a430fd36312a01047
Parents: 64aa9ea
Author: sboikov <sb...@gridgain.com>
Authored: Wed Dec 13 16:34:47 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Dec 20 11:00:19 2017 +0300
----------------------------------------------------------------------
.../zk/internal/ZookeeperDiscoveryImpl.java | 84 +++++++++++++++++---
1 file changed, 73 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f9698f4d/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 7032fd8..a72d742 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -93,6 +93,12 @@ public class ZookeeperDiscoveryImpl {
static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD";
/** */
+ private static final String IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS = "IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS";
+
+ /** */
+ private static final String IGNITE_ZOOKEEPER_DISCOVERY_EVTS_THROTTLE = "IGNITE_ZOOKEEPER_DISCOVERY_EVTS_THROTTLE";
+
+ /** */
final ZookeeperDiscoverySpi spi;
/** */
@@ -1153,23 +1159,20 @@ public class ZookeeperDiscoveryImpl {
TreeMap<Integer, String> alives = new TreeMap<>();
- TreeMap<Long, ZookeeperClusterNode> curTop = new TreeMap<>(rtState.top.nodesByOrder);
-
- boolean newEvts = false;
-
for (String child : aliveNodes) {
Integer internalId = ZkIgnitePaths.aliveInternalId(child);
Object old = alives.put(internalId, child);
assert old == null;
-
- if (!rtState.top.nodesByInternalId.containsKey(internalId)) {
- if (processJoinOnCoordinator(curTop, internalId, child))
- newEvts = true;
- }
}
+ TreeMap<Long, ZookeeperClusterNode> curTop = new TreeMap<>(rtState.top.nodesByOrder);
+
+ int newEvts = 0;
+
+ final int MAX_NEW_EVTS = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS, 100);
+
List<ZookeeperClusterNode> failedNodes = null;
for (Map.Entry<Integer, ZookeeperClusterNode> e : rtState.top.nodesByInternalId.entrySet()) {
@@ -1183,11 +1186,51 @@ public class ZookeeperDiscoveryImpl {
generateNodeFail(curTop, failedNode);
- newEvts = true;
+ newEvts++;
+
+ if (newEvts == MAX_NEW_EVTS) {
+ saveAndProcessNewEvents();
+
+ if (log.isInfoEnabled()) {
+ log.info("Delay alive nodes change process, max event threshold reached [newEvts=" + newEvts +
+ ", totalEvts=" + rtState.evtsData.evts.size() + ']');
+ }
+
+ throttleNewEventsGeneration();
+
+ rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+ return;
+ }
}
}
- if (newEvts)
+ for (Map.Entry<Integer, String> e : alives.entrySet()) {
+ Integer internalId = e.getKey();
+
+ if (!rtState.top.nodesByInternalId.containsKey(internalId)) {
+ if (processJoinOnCoordinator(curTop, internalId, e.getValue())) {
+ newEvts++;
+
+ if (newEvts == MAX_NEW_EVTS) {
+ saveAndProcessNewEvents();
+
+ if (log.isInfoEnabled()) {
+ log.info("Delay alive nodes change process, max event threshold reached [newEvts=" + newEvts +
+ ", totalEvts=" + rtState.evtsData.evts.size() + ']');
+ }
+
+ throttleNewEventsGeneration();
+
+ rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+ return;
+ }
+ }
+ }
+ }
+
+ if (newEvts > 0)
saveAndProcessNewEvents();
if (failedNodes != null)
@@ -1195,6 +1238,25 @@ public class ZookeeperDiscoveryImpl {
}
/**
+ *
+ */
+ private void throttleNewEventsGeneration() {
+ long delay = IgniteSystemProperties.getLong(IGNITE_ZOOKEEPER_DISCOVERY_EVTS_THROTTLE, 0);
+
+ if (delay > 0) {
+ if (log.isInfoEnabled())
+ log.info("Sleep delay before generate new events [delay=" + delay + ']');
+
+ try {
+ Thread.sleep(delay);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ /**
* @param nodeId Node ID.
* @param prefixId Path prefix.
* @return Join data.
[38/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9e4f2c20
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9e4f2c20
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9e4f2c20
Branch: refs/heads/ignite-zk
Commit: 9e4f2c20c2582655637c5abf349abc9e5473e9fe
Parents: fc085a4
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 18 15:25:34 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 18 18:08:47 2017 +0300
----------------------------------------------------------------------
.../internal/ZkCommunicationErrorNodeState.java | 8 +-
.../ZkCommunicationErrorProcessFuture.java | 28 ++
...kCommunicationErrorResolveFinishMessage.java | 7 +-
.../ZkCommunicationErrorResolveResult.java | 15 +-
.../internal/ZkCommunicationProblemContext.java | 67 +++-
.../zk/internal/ZkDiscoveryEventsData.java | 2 +-
.../ZkDistributedCollectDataFuture.java | 11 +-
.../discovery/zk/internal/ZookeeperClient.java | 2 +-
.../zk/internal/ZookeeperDiscoveryImpl.java | 359 +++++++++++++++----
.../ZookeeperDiscoverySpiBasicTest.java | 138 ++++++-
10 files changed, 540 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
index ddc310d..9c21f13 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
@@ -23,21 +23,23 @@ import java.util.BitSet;
/**
*
*/
-public class ZkCommunicationErrorNodeState implements Serializable {
+class ZkCommunicationErrorNodeState implements Serializable {
/** */
private static final long serialVersionUID = 0L;
/** */
- private final BitSet commState;
+ final BitSet commState;
/** */
- private final Exception err;
+ final Exception err;
/**
* @param commState Communication state.
* @param err Error if failed get communication state..
*/
ZkCommunicationErrorNodeState(BitSet commState, Exception err) {
+ assert commState != null || err != null;
+
this.commState = commState;
this.err = err;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index a6294bd..0074817 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -214,6 +214,34 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
}
/**
+ * @param err Error.
+ */
+ void onError(Exception err) {
+ assert err != null;
+
+ Map<Long, GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ if (state == State.DONE) {
+ assert resErr != null;
+
+ return;
+ }
+
+ state = State.DONE;
+
+ resErr = err;
+
+ futs = nodeFuts; // nodeFuts should not be modified after state changed to DONE.
+ }
+
+ for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : futs.entrySet())
+ e.getValue().onDone(err);
+
+ onDone(err);
+ }
+
+ /**
* @param failedNodes Node failed as result of resolve process.
*/
void onFinishResolve(Set<Long> failedNodes) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
index 20aeddf..147b78f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
@@ -33,13 +33,18 @@ class ZkCommunicationErrorResolveFinishMessage implements DiscoverySpiCustomMess
final UUID futId;
/** */
+ final long topVer;
+
+ /** */
transient ZkCommunicationErrorResolveResult res;
/**
* @param futId Future ID.
+ * @param topVer Topology version when resolve process finished.
*/
- ZkCommunicationErrorResolveFinishMessage(UUID futId) {
+ ZkCommunicationErrorResolveFinishMessage(UUID futId, long topVer) {
this.futId = futId;
+ this.topVer = topVer;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
index 607f93b..68cbdb8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
@@ -18,7 +18,7 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.io.Serializable;
-import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridIntList;
import org.jetbrains.annotations.Nullable;
/**
@@ -29,12 +29,17 @@ class ZkCommunicationErrorResolveResult implements Serializable {
private static final long serialVersionUID = 0L;
/** */
- final GridLongList failedNodes;
+ final GridIntList killedNodes;
+
+ /** */
+ final Exception err;
/**
- * @param failedNodes
+ * @param killedNodes Killed nodes.
+ * @param err Error.
*/
- ZkCommunicationErrorResolveResult(@Nullable GridLongList failedNodes) {
- this.failedNodes = failedNodes;
+ ZkCommunicationErrorResolveResult(@Nullable GridIntList killedNodes, Exception err) {
+ this.killedNodes = killedNodes;
+ this.err = err;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
index fd11b55..9cb48cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationProblemContext.java
@@ -17,9 +17,14 @@
package org.apache.ignite.spi.discovery.zk.internal;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.UUID;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CommunicationProblemContext;
@@ -28,31 +33,73 @@ import org.apache.ignite.configuration.CommunicationProblemContext;
*/
class ZkCommunicationProblemContext implements CommunicationProblemContext {
/** */
+ private static final Comparator<ClusterNode> NODE_ORDER_CMP = new Comparator<ClusterNode>() {
+ @Override public int compare(ClusterNode node1, ClusterNode node2) {
+ return Long.compare(node1.order(), node2.order());
+ }
+ };
+
+ /** */
private Set<ClusterNode> killedNodes = new HashSet<>();
+ /** */
+ private final Map<UUID, BitSet> nodesState;
+
+ /** */
+ private final List<ClusterNode> initialNodes;
+
+ /** */
+ private final List<ClusterNode> curNodes;
+
+ /**
+ * @param curNodes Current topology snapshot.
+ * @param initialNodes Topology snapshot when communication error resolve started.
+ * @param nodesState Nodes communication state.
+ */
+ ZkCommunicationProblemContext(List<ClusterNode> curNodes,
+ List<ClusterNode> initialNodes,
+ Map<UUID, BitSet> nodesState)
+ {
+ this.curNodes = Collections.unmodifiableList(curNodes);
+ this.initialNodes = initialNodes;
+ this.nodesState = nodesState;
+ }
+
/** {@inheritDoc} */
@Override public List<ClusterNode> topologySnapshot() {
- return null;
+ return curNodes;
}
/** {@inheritDoc} */
@Override public boolean connectionAvailable(ClusterNode node1, ClusterNode node2) {
- return false;
+ BitSet nodeState = nodesState.get(node1.id());
+
+ if (nodeState == null)
+ throw new IllegalArgumentException("Invalid node: " + node1);
+
+ int nodeIdx = Collections.binarySearch(initialNodes, node2, NODE_ORDER_CMP);
+
+ if (nodeIdx < 0)
+ throw new IllegalArgumentException("Invalid node: " + node2);
+
+ assert nodeIdx < nodeState.size() : nodeIdx;
+
+ return nodeState.get(nodeIdx);
}
/** {@inheritDoc} */
@Override public List<String> startedCaches() {
- return null;
+ return null; // TODO ZK
}
/** {@inheritDoc} */
@Override public List<List<ClusterNode>> cacheAffinity(String cacheName) {
- return null;
+ return null; // TODO ZK
}
/** {@inheritDoc} */
@Override public List<List<ClusterNode>> cachePartitionOwners(String cacheName) {
- return null;
+ return null; // TODO ZK
}
/** {@inheritDoc} */
@@ -60,6 +107,16 @@ class ZkCommunicationProblemContext implements CommunicationProblemContext {
if (node == null)
throw new NullPointerException();
+ if (Collections.binarySearch(curNodes, node, NODE_ORDER_CMP) < 0)
+ throw new IllegalArgumentException("Invalid node: " + node);
+
killedNodes.add(node);
}
+
+ /**
+ * @return Nodes to fail.
+ */
+ Set<ClusterNode> killedNodes() {
+ return killedNodes;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
index faea49e..6bdf573 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
@@ -69,7 +69,7 @@ class ZkDiscoveryEventsData implements Serializable {
/**
* @param id Future ID.
*/
- void communicationErrorResolveFutureId(UUID id) {
+ void communicationErrorResolveFutureId(@Nullable UUID id) {
commErrFutId = id;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
index 19e2acc..2467928 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -141,9 +141,14 @@ class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
// TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
String evtDir = paths.distributedFutureBasePath(futId);
- client.deleteAll(evtDir,
- client.getChildren(evtDir),
- -1);
+ try {
+ client.deleteAll(evtDir,
+ client.getChildren(evtDir),
+ -1);
+ }
+ catch (KeeperException.NoNodeException e) {
+ // TODO ZK
+ }
client.deleteIfExists(evtDir, -1);
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index 5923b39..5c0bd58 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -940,7 +940,7 @@ public class ZookeeperClient implements Watcher {
else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
else
- assert rc == 0 : rc;
+ assert rc == 0 : KeeperException.Code.get(rc);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 62fc581..ef3504f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -19,6 +19,7 @@ package org.apache.ignite.spi.discovery.zk.internal;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
@@ -50,6 +51,7 @@ import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.util.GridIntList;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -246,6 +248,8 @@ public class ZookeeperDiscoveryImpl {
* @param err Connect error.
*/
public void onCommunicationConnectionError(ClusterNode node0, Exception err) {
+ checkState();
+
ZookeeperClusterNode node = node(node0.id());
if (node == null)
@@ -269,7 +273,16 @@ public class ZookeeperDiscoveryImpl {
", err= " + err + ']');
}
- fut.scheduleCheckOnTimeout();
+ ConnectionState connState;
+
+ synchronized (this) {
+ connState = this.connState;
+ }
+
+ if (connState != ConnectionState.STARTED)
+ fut.onError(new IgniteCheckedException("Node stopped."));
+ else
+ fut.scheduleCheckOnTimeout();
}
else
fut = commErrProcFut.get();
@@ -475,7 +488,7 @@ public class ZookeeperDiscoveryImpl {
break;
case STOPPED:
- throw new IgniteSpiException("Zookeeper client closed.");
+ throw new IgniteSpiException("Node stopped.");
case DISCONNECTED:
throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
@@ -1025,11 +1038,41 @@ public class ZookeeperDiscoveryImpl {
}
/**
+ * @param lastEvts Last events from previous coordinator.
+ * @throws Exception If failed.
+ */
+ private void previousCoordinatorCleanup(ZkDiscoveryEventsData lastEvts) throws Exception {
+ for (ZkDiscoveryEventData evtData : lastEvts.evts.values()) {
+ if (evtData instanceof ZkDiscoveryCustomEventData) {
+ ZkDiscoveryCustomEventData evtData0 = (ZkDiscoveryCustomEventData)evtData;
+
+ // It is possible previous coordinator failed before finished message processing.
+ if (evtData0.msg instanceof ZkCommunicationErrorResolveFinishMessage) {
+ try {
+ ZkCommunicationErrorResolveFinishMessage msg =
+ (ZkCommunicationErrorResolveFinishMessage)evtData0.msg;
+
+ ZkCommunicationErrorResolveResult res = unmarshalZip(
+ ZkDistributedCollectDataFuture.readResult(rtState.zkClient, zkPaths, msg.futId));
+
+ deleteAliveNodes(res.killedNodes);
+ }
+ catch (KeeperException.NoNodeException ignore) {
+ // No-op.
+ }
+ }
+ }
+ }
+ }
+
+ /**
* @param aliveNodes Alive nodes paths.
* @param locInternalId Local node's internal ID.
* @throws Exception If failed.
*/
private void onBecomeCoordinator(List<String> aliveNodes, int locInternalId) throws Exception {
+ long topVer0 = rtState.evtsData != null ? rtState.evtsData.topVer : -1L;
+
byte[] evtsDataBytes = rtState.zkClient.getData(zkPaths.evtsPath);
if (evtsDataBytes.length > 0)
@@ -1044,6 +1087,11 @@ public class ZookeeperDiscoveryImpl {
assert locNode.order() > 0 : locNode;
assert rtState.evtsData != null;
+ previousCoordinatorCleanup(rtState.evtsData);
+
+ if (topVer0 > rtState.evtsData.topVer)
+ rtState.evtsData.topVer = topVer0;
+
UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
if (futId != null) {
@@ -1636,6 +1684,7 @@ public class ZookeeperDiscoveryImpl {
assert node != null : msg0.nodeId;
+ // TODO ZK: delete when process event
for (String child : zkClient.getChildren(zkPaths.aliveNodesDir)) {
if (ZkIgnitePaths.aliveInternalId(child) == node.internalId()) {
zkClient.deleteIfExistsAsync(zkPaths.aliveNodesDir + "/" + child);
@@ -2021,22 +2070,25 @@ public class ZookeeperDiscoveryImpl {
ClusterNode creatorNode = rtState.top.nodesById.get(evtData.sndNodeId);
if (msg.warning != null) {
- U.warn(log, "Received EVT_NODE_FAILED event with warning [" +
- "nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) +
- ", nodeId=" + msg.nodeId +
- ", msg=" + msg.warning + ']');
+ U.warn(log, "Received force EVT_NODE_FAILED event with warning [" +
+ "nodeId=" + msg.nodeId +
+ ", msg=" + msg.warning +
+ ", nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) + ']');
}
else {
U.warn(log, "Received force EVT_NODE_FAILED event [" +
- "nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) +
- ", nodeId=" + msg.nodeId + ']');
+ "nodeId=" + msg.nodeId +
+ ", nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) + ']');
}
ZookeeperClusterNode node = rtState.top.nodesById.get(msg.nodeId);
assert node != null : msg.nodeId;
- processNodeFail(node.internalId(), evtData.topologyVersion());
+ if (node.isLocal())
+ throw localNodeFail("Received force EVT_NODE_FAILED event for local node.");
+ else
+ notifyNodeFail(node.internalId(), evtData.topologyVersion());
}
/**
@@ -2049,13 +2101,10 @@ public class ZookeeperDiscoveryImpl {
UUID futId = msg.futId;
assert futId != null;
- assert futId.equals(rtState.evtsData.communicationErrorResolveFutureId());
if (log.isInfoEnabled())
log.info("Received communication error resolve finish message [reqId=" + futId + ']');
- rtState.evtsData.communicationErrorResolveFutureId(null);
-
rtState.commErrProcNodes = null;
ZkCommunicationErrorResolveResult res = msg.res;
@@ -2069,14 +2118,72 @@ public class ZookeeperDiscoveryImpl {
Set<Long> failedNodes = null;
- if (res.failedNodes != null) {
- failedNodes = U.newHashSet(res.failedNodes.size());
+ if (res.err != null)
+ U.error(log, "Communication error resolve failed: " + res.err, res.err);
+ else {
+ if (res.killedNodes != null) {
+ failedNodes = U.newHashSet(res.killedNodes.size());
+
+ for (int i = 0; i < res.killedNodes.size(); i++) {
+ int internalId = res.killedNodes.get(i);
+
+ if (internalId == locNode.internalId()) {
+ fut.onError(new IgniteCheckedException("Local node is forced to stop " +
+ "by communication error resolver"));
+
+ if (rtState.crd)
+ deleteAliveNodes(res.killedNodes);
+
+ throw localNodeFail("Local node is forced to stop by communication error resolver " +
+ "[nodeId=" + locNode.id() + ']');
+ }
+
+ ZookeeperClusterNode node = rtState.top.nodesByInternalId.get(internalId);
+
+ assert node != null : internalId;
+
+ failedNodes.add(node.order());
+ }
+
+ long topVer = msg.topVer;
+
+ for (int i = 0; i < res.killedNodes.size(); i++) {
+ int nodeInternalId = res.killedNodes.get(i);
- for (int i = 0; i < res.failedNodes.size(); i++)
- failedNodes.add(res.failedNodes.get(i));
+ ClusterNode node = rtState.top.nodesByInternalId.get(nodeInternalId);
+
+ assert node != null : nodeInternalId;
+
+ if (log.isInfoEnabled())
+ log.info("Node stop is forced by communication error resolver [nodeId=" + node.id() + ']');
+
+ notifyNodeFail(nodeInternalId, ++topVer);
+ }
+ }
}
fut.onFinishResolve(failedNodes);
+
+ if (rtState.crd)
+ deleteAliveNodes(res.killedNodes);
+ }
+
+ /**
+ * @param internalIds Nodes internal IDs.
+ * @throws Exception If failed.
+ */
+ private void deleteAliveNodes(@Nullable GridIntList internalIds) throws Exception {
+ if (internalIds == null)
+ return;
+
+ List<String> alives = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
+
+ for (int i = 0; i < alives.size(); i++) {
+ String alive = alives.get(i);
+
+ if (internalIds.contains(ZkIgnitePaths.aliveInternalId(alive)))
+ rtState.zkClient.deleteIfExistsAsync(zkPaths.aliveNodesDir + "/" + alive);
+ }
}
/**
@@ -2163,7 +2270,7 @@ public class ZookeeperDiscoveryImpl {
private void onCommunicationErrorResolveStatusReceived(ZkRuntimeState rtState) throws Exception {
ZkDiscoveryEventsData evtsData = rtState.evtsData;
- UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
+ UUID futId = evtsData.communicationErrorResolveFutureId();
if (log.isInfoEnabled())
log.info("Received communication status from all nodes [reqId=" + futId + ']');
@@ -2178,43 +2285,113 @@ public class ZookeeperDiscoveryImpl {
rtState.commErrProcNodes = null;
- ZkClusterNodes top = rtState.top;
+ List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
- List<ZkCommunicationErrorNodeState> nodesRes = new ArrayList<>();
+ Map<UUID, BitSet> nodesRes = U.newHashMap(topSnapshot.size());
- for (ZookeeperClusterNode node : top.nodesByOrder.values()) {
+ Exception err = null;
+
+ for (ClusterNode node : topSnapshot) {
byte[] stateBytes = ZkDistributedCollectDataFuture.readNodeResult(futPath,
rtState.zkClient,
node.order());
ZkCommunicationErrorNodeState nodeState = unmarshalZip(stateBytes);
- nodesRes.add(nodeState);
+ if (nodeState.err != null) {
+ if (err == null)
+ err = new Exception("Failed to resolve communication error.");
+
+ err.addSuppressed(nodeState.err);
+ }
+ else {
+ assert nodeState.commState != null;
+
+ nodesRes.put(node.id(), nodeState.commState);
+ }
}
- ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId);
+ long topVer = evtsData.topVer;
- ZkCommunicationErrorResolveResult res = new ZkCommunicationErrorResolveResult(null);
+ GridIntList killedNodesList = null;
- msg.res = res;
+ if (err == null) {
+ boolean fullyConnected = true;
- ZkDistributedCollectDataFuture.saveResult(zkPaths.distributedFutureResultPath(futId),
- rtState.zkClient,
- marshalZip(res));
+ for (Map.Entry<UUID, BitSet> e : nodesRes.entrySet()) {
+ if (!checkFullyConnected(e.getValue(), initialNodes, rtState.top)) {
+ fullyConnected = false;
+
+ break;
+ }
+ }
- CommunicationProblemResolver rslvr = spi.ignite().configuration().getCommunicationProblemResolver();
+ if (fullyConnected) {
+ if (log.isInfoEnabled()) {
+ log.info("Finish communication error resolve process automatically, there are no " +
+ "communication errors [reqId=" + futId + ']');
+ }
+ }
+ else {
+ CommunicationProblemResolver rslvr = spi.ignite().configuration().getCommunicationProblemResolver();
+
+ if (rslvr != null) {
+ if (log.isInfoEnabled()) {
+ log.info("Call communication error resolver [reqId=" + futId +
+ ", rslvr=" + rslvr.getClass().getSimpleName() + ']');
+ }
- if (rslvr != null) {
- ZkCommunicationProblemContext ctx = new ZkCommunicationProblemContext();
+ ZkCommunicationProblemContext ctx = new ZkCommunicationProblemContext(topSnapshot,
+ initialNodes,
+ nodesRes);
- rslvr.resolve(ctx);
+ try {
+ rslvr.resolve(ctx);
+
+ Set<ClusterNode> killedNodes = ctx.killedNodes();
+
+ if (killedNodes != null) {
+ if (log.isInfoEnabled()) {
+ log.info("Communication error resolver forces nodes stop [reqId=" + futId +
+ ", killNodeCnt=" + killedNodes.size() +
+ ", nodeIds=" + U.nodeIds(killedNodes) + ']');
+ }
+
+ killedNodesList = new GridIntList(killedNodes.size());
+
+ for (ClusterNode killedNode : killedNodes) {
+ killedNodesList.add(((ZookeeperClusterNode)killedNode).internalId());
+
+ evtsData.topVer++;
+ }
+ }
+ }
+ catch (Exception e) {
+ err = e;
+
+ U.error(log, "Failed to resolve communication error with configured resolver [reqId=" + futId + ']', e);
+ }
+ }
+ }
}
+ evtsData.communicationErrorResolveFutureId(null);
+
+ ZkCommunicationErrorResolveResult res = new ZkCommunicationErrorResolveResult(killedNodesList, err);
+
+ ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId, topVer);
+
+ msg.res = res;
+
+ ZkDistributedCollectDataFuture.saveResult(zkPaths.distributedFutureResultPath(futId),
+ rtState.zkClient,
+ marshalZip(res));
+
evtsData.evtIdGen++;
ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
evtsData.evtIdGen,
- evtsData.topVer,
+ topVer,
locNode.id(),
msg,
null,
@@ -2228,6 +2405,30 @@ public class ZookeeperDiscoveryImpl {
}
/**
+ * @param commState Node communication state.
+ * @param initialNodes Topology snapshot when communication error resolve started.
+ * @param top Current topology.
+ * @return {@code True} if node has connection to all alive nodes.
+ */
+ private boolean checkFullyConnected(BitSet commState, List<ClusterNode> initialNodes, ZkClusterNodes top) {
+ int startIdx = 0;
+
+ for (;;) {
+ int idx = commState.nextClearBit(startIdx);
+
+ if (idx >= initialNodes.size())
+ return true;
+
+ ClusterNode node = initialNodes.get(idx);
+
+ if (top.nodesById.containsKey(node.id()))
+ return false;
+
+ startIdx = idx + 1;
+ }
+ }
+
+ /**
*
*/
public void simulateNodeFailure() {
@@ -2292,70 +2493,73 @@ public class ZookeeperDiscoveryImpl {
/**
* @param evtData Event data.
- * @throws Exception If failed.
*/
- private void notifyNodeFail(final ZkDiscoveryNodeFailEventData evtData) throws Exception {
- processNodeFail(evtData.failedNodeInternalId(), evtData.topologyVersion());
+ private void notifyNodeFail(final ZkDiscoveryNodeFailEventData evtData) {
+ notifyNodeFail(evtData.failedNodeInternalId(), evtData.topologyVersion());
}
/**
- * @param nodeInternalId Failed node internal ID.
+ * @param nodeInternalOrder Node order.
* @param topVer Topology version.
- * @throws Exception If failed.
*/
- private void processNodeFail(int nodeInternalId, long topVer) throws Exception {
- final ZookeeperClusterNode failedNode = rtState.top.removeNode(nodeInternalId);
+ private void notifyNodeFail(int nodeInternalOrder, long topVer) {
+ final ZookeeperClusterNode failedNode = rtState.top.removeNode(nodeInternalOrder);
- assert failedNode != null;
+ assert failedNode != null && !failedNode.isLocal() : failedNode;
- if (failedNode.isLocal()) {
- U.warn(log, "Received EVT_NODE_FAILED for local node.");
+ PingFuture pingFut = pingFuts.get(failedNode.order());
- rtState.onCloseStart();
+ if (pingFut != null)
+ pingFut.onDone(false);
- if (locNode.isClient() && clientReconnectEnabled) {
- boolean reconnect = false;
+ final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
- synchronized (stateMux) {
- if (connState == ConnectionState.STARTED) {
- reconnect = true;
+ lsnr.onDiscovery(EVT_NODE_FAILED,
+ topVer,
+ failedNode,
+ topSnapshot,
+ Collections.<Long, Collection<ClusterNode>>emptyMap(),
+ null);
+ }
- connState = ConnectionState.DISCONNECTED;
- }
- }
+ /**
+ * @param msg Message to log.
+ * @return Exception to be thrown.
+ */
+ private Exception localNodeFail(String msg) {
+ U.warn(log, msg);
+
+ rtState.onCloseStart();
- if (reconnect) {
- UUID newId = UUID.randomUUID();
+ if (clientReconnectEnabled) {
+ assert locNode.isClient() : locNode;
- U.quietAndWarn(log, "Received EVT_NODE_FAILED for local node, will try to reconnect with new id [" +
- "newId=" + newId +
- ", prevId=" + locNode.id() +
- ", locNode=" + locNode + ']');
+ boolean reconnect = false;
- runInWorkerThread(new ReconnectClosure(newId));
+ synchronized (stateMux) {
+ if (connState == ConnectionState.STARTED) {
+ reconnect = true;
+
+ connState = ConnectionState.DISCONNECTED;
}
}
- else
- notifySegmented();
- // Stop any further processing.
- throw new ZookeeperClientFailedException("Received node failed event for local node.");
- }
- else {
- PingFuture pingFut = pingFuts.get(failedNode.order());
-
- if (pingFut != null)
- pingFut.onDone(false);
+ if (reconnect) {
+ UUID newId = UUID.randomUUID();
- final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+ U.quietAndWarn(log, "Client node will try to reconnect with new id [" +
+ "newId=" + newId +
+ ", prevId=" + locNode.id() +
+ ", locNode=" + locNode + ']');
- lsnr.onDiscovery(EVT_NODE_FAILED,
- topVer,
- failedNode,
- topSnapshot,
- Collections.<Long, Collection<ClusterNode>>emptyMap(),
- null);
+ runInWorkerThread(new ReconnectClosure(newId));
+ }
}
+ else
+ notifySegmented();
+
+ // Stop any further processing.
+ return new ZookeeperClientFailedException(msg);
}
/**
@@ -2629,6 +2833,11 @@ public class ZookeeperDiscoveryImpl {
if (zkClient != null)
zkClient.close();
+ ZkCommunicationErrorProcessFuture commErrFut = commErrProcFut.get();
+
+ if (commErrFut != null)
+ commErrFut.onError(new IgniteCheckedException("Node stopped."));
+
IgniteUtils.shutdownNow(ZookeeperDiscoveryImpl.class, utilityPool, log);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/9e4f2c20/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index cee2e76..a29f478 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -20,7 +20,9 @@ package org.apache.ignite.spi.discovery.zk.internal;
import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -62,6 +64,7 @@ import org.apache.ignite.internal.IgnitionEx;
import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -1712,6 +1715,8 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * TODO ZK
+ *
* @throws Exception If failed.
*/
public void _testCommunicationFailure() throws Exception {
@@ -1918,6 +1923,97 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testCommunicationErrorResolve_KillNode_1() throws Exception {
+ communicationErrorResolve_KillNodes(2, Collections.singletonList(2L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillNode_2() throws Exception {
+ communicationErrorResolve_KillNodes(3, Collections.singletonList(2L));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillNode_3() throws Exception {
+ communicationErrorResolve_KillNodes(10, Arrays.asList(2L, 4L, 6L));
+ }
+
+ /**
+ * @param startNodes Number of nodes to start.
+ * @param killNodes Nodes to kill by resolve process.
+ * @throws Exception If failed.
+ */
+ private void communicationErrorResolve_KillNodes(int startNodes, Collection<Long> killNodes) throws Exception {
+ testCommSpi = true;
+
+ commProblemRslvr = new TestNodeKillCommunicationProblemResolver(killNodes);
+
+ startGrids(startNodes);
+
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(0));
+
+ commSpi.checkRes = new BitSet(startNodes);
+
+ ZookeeperDiscoverySpi spi = null;
+
+ for (Ignite node : G.allGrids()) {
+ ZookeeperDiscoverySpi spi0 = spi(node);
+
+ if (!killNodes.contains(node.cluster().localNode().order())) {
+ spi = spi0;
+
+ break;
+ }
+ }
+
+ assertNotNull(spi);
+
+ spi.onCommunicationConnectionError(ignite(1).cluster().localNode(), new Exception("test"));
+
+ int expNodes = startNodes - killNodes.size();
+
+ waitForTopology(expNodes);
+
+ for (Ignite node : G.allGrids())
+ assertFalse(killNodes.contains(node.cluster().localNode().order()));
+
+ startGrid(startNodes);
+
+ waitForTopology(expNodes + 1);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCommunicationErrorResolve_KillCoordinator() throws Exception {
+ // Kill coordinator.
+ testCommSpi = true;
+
+ commProblemRslvr = new TestNodeKillCommunicationProblemResolver(Collections.singleton(1L));
+
+ startGrids(3);
+
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(2));
+
+ commSpi.checkRes = new BitSet(3);
+
+ ZookeeperDiscoverySpi spi = spi(ignite(1));
+
+ spi.onCommunicationConnectionError(ignite(0).cluster().localNode(), new Exception("test"));
+
+ waitForTopology(2);
+
+ startGrid(10);
+
+ waitForTopology(3);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testConnectionCheck() throws Exception {
final int NODES = 5;
@@ -1928,9 +2024,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
TcpCommunicationSpi spi = (TcpCommunicationSpi)node.configuration().getCommunicationSpi();
- List<ClusterNode> nodes = new ArrayList<>();
-
- nodes.addAll(node.cluster().nodes());
+ List<ClusterNode> nodes = new ArrayList<>(node.cluster().nodes());
BitSet res = spi.checkConnection(nodes).get();
@@ -2622,10 +2716,40 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
*
*/
+ static class TestNodeKillCommunicationProblemResolver implements CommunicationProblemResolver {
+ /** */
+ final Collection<Long> killNodeOrders;
+
+ /**
+ * @param killOrders Killed nodes order.
+ */
+ TestNodeKillCommunicationProblemResolver(Collection<Long> killOrders) {
+ this.killNodeOrders = killOrders;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolve(CommunicationProblemContext ctx) {
+ List<ClusterNode> nodes = ctx.topologySnapshot();
+
+ assertTrue(nodes.size() > 0);
+
+ for (ClusterNode node : nodes) {
+ if (killNodeOrders.contains(node.order()))
+ ctx.killNode(node);
+ }
+ }
+ }
+
+ /**
+ *
+ */
static class ZkTestCommunicationSpi extends TcpCommunicationSpi {
/** */
private volatile CountDownLatch pingLatch;
+ /** */
+ private volatile BitSet checkRes;
+
/**
* @param ignite Node.
* @return Node's communication SPI.
@@ -2646,6 +2770,14 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
throw new IgniteException(e);
}
+ BitSet checkRes = this.checkRes;
+
+ if (checkRes != null) {
+ this.checkRes = null;
+
+ return new IgniteFinishedFutureImpl<>(checkRes);
+ }
+
return super.checkConnection(nodes);
}
}
[30/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk-ce
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk-ce
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
# modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/78a994a9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/78a994a9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/78a994a9
Branch: refs/heads/ignite-zk
Commit: 78a994a98800120680bcf58b727e2978bffeb531
Parents: 60b625a e358ae2
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 14:25:10 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 14:25:10 2017 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 9 +-
.../spi/communication/tcp/HandshakeMessage.java | 156 ------------------
.../communication/tcp/HandshakeMessage2.java | 95 -----------
.../spi/communication/tcp/NodeIdMessage.java | 115 --------------
.../tcp/RecoveryLastReceivedMessage.java | 113 -------------
.../communication/tcp/TcpCommunicationSpi.java | 31 ++--
.../tcp/messages/HandshakeMessage.java | 157 +++++++++++++++++++
.../tcp/messages/HandshakeMessage2.java | 95 +++++++++++
.../tcp/messages/NodeIdMessage.java | 128 +++++++++++++++
.../messages/RecoveryLastReceivedMessage.java | 114 ++++++++++++++
.../tcp/messages/package-info.java | 22 +++
11 files changed, 541 insertions(+), 494 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/78a994a9/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/78a994a9/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index d4ccc33,69da9ca..3588a79
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@@ -135,7 -132,10 +135,11 @@@ import org.apache.ignite.spi.IgniteSpiT
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.CommunicationListener;
import org.apache.ignite.spi.communication.CommunicationSpi;
+ import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage;
+ import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
+ import org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage;
+ import org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.thread.IgniteThread;
[15/50] [abbrv] ignite git commit: IGNITE-7199 Web console: several
minor improvements.
Posted by sb...@apache.org.
IGNITE-7199 Web console: several minor improvements.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e08d19cb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e08d19cb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e08d19cb
Branch: refs/heads/ignite-zk
Commit: e08d19cb785f2cba94c99d2dd4df14310ef2e8d6
Parents: 3325876
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Dec 14 12:07:40 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Dec 14 12:07:40 2017 +0700
----------------------------------------------------------------------
modules/web-console/backend/.eslintrc | 185 -------------------
.../web-console/backend/app/agentsHandler.js | 20 +-
.../web-console/backend/app/browsersHandler.js | 8 +-
modules/web-console/backend/app/mongo.js | 4 +
.../backend/config/settings.json.sample | 8 +-
.../1508395969410-init-registered-date.js | 33 ++++
modules/web-console/backend/package.json | 2 +
modules/web-console/backend/services/Utils.js | 51 +++++
modules/web-console/backend/services/auth.js | 26 +--
modules/web-console/backend/services/caches.js | 4 +
.../web-console/backend/services/clusters.js | 4 +
modules/web-console/backend/services/domains.js | 4 +
modules/web-console/backend/services/igfss.js | 4 +
modules/web-console/backend/services/mails.js | 80 ++++----
.../web-console/backend/services/notebooks.js | 4 +
modules/web-console/backend/services/users.js | 23 +--
modules/web-console/frontend/app/app.js | 6 +
.../app/components/bs-select-menu/style.scss | 10 +-
.../app/components/bs-select-menu/template.pug | 15 +-
.../app/components/grid-export/component.js | 52 ++++++
.../app/components/grid-export/index.js | 24 +++
.../app/components/grid-export/template.pug | 18 ++
.../app/components/grid-no-data/component.js | 33 ++++
.../app/components/grid-no-data/controller.js | 50 +++++
.../app/components/grid-no-data/index.js | 24 +++
.../app/components/grid-no-data/style.scss | 31 ++++
.../list-editable-cols/cols.template.pug | 1 -
.../list-editable-cols/row.directive.js | 2 +-
.../app/components/list-editable/style.scss | 12 +-
.../app/components/list-editable/template.pug | 6 +-
.../list-of-registered-users.tpl.pug | 2 +-
.../app/components/ui-grid-filters/directive.js | 62 +++++++
.../app/components/ui-grid-filters/index.js | 43 +++++
.../app/components/ui-grid-filters/style.scss | 36 ++++
.../app/components/ui-grid-filters/template.pug | 47 +++++
.../web-console-header-extension/component.js | 22 +++
.../web-console-header-extension/template.pug | 15 ++
.../app/components/web-console-header/index.js | 4 +-
.../components/web-console-header/style.scss | 2 +-
.../components/web-console-header/template.pug | 6 +-
.../app/modules/form/field/input/text.scss | 1 +
.../frontend/app/modules/user/permissions.js | 4 +-
.../frontend/app/primitives/btn/index.scss | 17 ++
.../app/primitives/form-field/index.scss | 2 +-
.../app/primitives/ui-grid-settings/index.scss | 12 ++
.../frontend/app/primitives/ui-grid/index.scss | 2 +
modules/web-console/frontend/package.json | 3 +-
.../frontend/public/images/checkbox-active.svg | 25 +++
.../frontend/public/images/checkbox.svg | 22 +++
.../frontend/public/images/icons/alert.svg | 1 +
.../frontend/public/images/icons/checkmark.svg | 3 +
.../frontend/public/images/icons/index.js | 4 +-
.../frontend/public/images/icons/sort.svg | 2 +-
.../frontend/views/includes/header-left.pug | 8 +-
.../frontend/views/includes/header-right.pug | 4 +-
.../frontend/views/settings/profile.tpl.pug | 9 +-
56 files changed, 790 insertions(+), 312 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/.eslintrc
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/.eslintrc b/modules/web-console/backend/.eslintrc
deleted file mode 100644
index 1915518..0000000
--- a/modules/web-console/backend/.eslintrc
+++ /dev/null
@@ -1,185 +0,0 @@
-env:
- es6: true
- node: true
- mocha: true
-
-parserOptions:
- sourceType: module
- ecmaFeatures:
- arrowFunctions: true
- blockBindings: true
- classes: true
- defaultParams: true
- destructuring: true
- module: true
- objectLiteralComputedProperties: true
- objectLiteralShorthandMethods: true
- objectLiteralShorthandProperties: true
- spread: true
- templateStrings: true
- experimentalObjectRestSpread: true
-
-globals:
- _: true
- io: true
-
-rules:
- arrow-parens: [1, "always"]
- arrow-spacing: [1, { "before": true, "after": true }]
- accessor-pairs: 2
- block-scoped-var: 2
- brace-style: [0, "1tbs"]
- comma-dangle: [2, "never"]
- comma-spacing: [2, {"before": false, "after": true}]
- comma-style: [2, "last"]
- complexity: [1, 40]
- computed-property-spacing: [2, "never"]
- consistent-return: 0
- consistent-this: [0, "that"]
- constructor-super: 2
- curly: [2, "multi-or-nest"]
- default-case: 2
- dot-location: 0
- dot-notation: [2, { "allowKeywords": true }]
- eol-last: 2
- eqeqeq: 2
- func-names: 0
- func-style: [0, "declaration"]
- generator-star-spacing: 0
- guard-for-in: 1
- handle-callback-err: 0
- id-length: [2, {"min": 1, "max": 60}]
- indent: [2, 4, {"SwitchCase": 1, "MemberExpression": "off", "CallExpression": {"arguments": "off"}}]
- key-spacing: [2, { "beforeColon": false, "afterColon": true }]
- lines-around-comment: 0
- linebreak-style: [0, "unix"]
- max-depth: [0, 4]
- max-len: [0, 120, 4]
- max-nested-callbacks: [1, 4]
- max-params: [0, 3]
- max-statements: [0, 10]
- new-cap: 2
- new-parens: 2
- no-alert: 2
- no-array-constructor: 2
- no-bitwise: 0
- no-caller: 2
- no-catch-shadow: 2
- no-cond-assign: 2
- no-console: 0
- no-constant-condition: 2
- no-continue: 0
- no-class-assign: 2
- no-const-assign: 2
- no-control-regex: 2
- no-debugger: 2
- no-delete-var: 2
- no-div-regex: 0
- no-dupe-keys: 2
- no-dupe-args: 2
- no-duplicate-case: 2
- no-else-return: 2
- no-empty: 2
- no-empty-character-class: 2
- no-eq-null: 2
- no-eval: 2
- no-ex-assign: 2
- no-extend-native: 2
- no-extra-bind: 2
- no-extra-boolean-cast: 2
- no-extra-parens: 0
- no-extra-semi: 2
- no-fallthrough: 2
- no-floating-decimal: 1
- no-func-assign: 2
- no-implied-eval: 2
- no-inline-comments: 0
- no-inner-declarations: [2, "functions"]
- no-invalid-regexp: 2
- no-irregular-whitespace: 2
- no-iterator: 2
- no-label-var: 2
- no-labels: 2
- no-lone-blocks: 2
- no-lonely-if: 2
- no-implicit-coercion: [2, {"boolean": false, "number": true, "string": true}]
- no-loop-func: 2
- no-mixed-requires: [0, false]
- no-mixed-spaces-and-tabs: [2, true]
- no-multi-spaces: ["error", {"exceptions": { "VariableDeclarator": true }}]
- no-multi-str: 2
- no-multiple-empty-lines: [0, {"max": 2}]
- no-native-reassign: 2
- no-negated-in-lhs: 2
- no-nested-ternary: 0
- no-new: 2
- no-new-func: 2
- no-new-object: 2
- no-new-require: 0
- no-new-wrappers: 2
- no-obj-calls: 2
- no-octal: 2
- no-octal-escape: 2
- no-param-reassign: 0
- no-path-concat: 0
- no-plusplus: 0
- no-process-env: 0
- no-process-exit: 0
- no-proto: 2
- no-redeclare: 2
- no-regex-spaces: 1
- no-restricted-modules: 0
- no-script-url: 0
- no-self-compare: 2
- no-sequences: 2
- no-shadow: 0
- no-shadow-restricted-names: 2
- no-spaced-func: 2
- no-sparse-arrays: 1
- no-sync: 0
- no-ternary: 0
- no-trailing-spaces: 2
- no-throw-literal: 0
- no-this-before-super: 2
- no-unexpected-multiline: 2
- no-undef: 2
- no-undef-init: 2
- no-undefined: 2
- no-unneeded-ternary: 2
- no-unreachable: 2
- no-unused-expressions: [2, { allowShortCircuit: true }]
- no-unused-vars: [2, {"vars": "all", "args": "after-used"}]
- no-use-before-define: 2
- no-useless-call: 2
- no-void: 0
- no-var: 2
- no-warning-comments: 0
- no-with: 2
- newline-after-var: 0
- object-shorthand: [2, "always"]
- one-var: [2, "never"]
- operator-assignment: [2, "always"]
- operator-linebreak: 0
- padded-blocks: 0
- prefer-const: 1
- prefer-spread: 2
- quote-props: [2, "as-needed"]
- quotes: [2, "single", {"allowTemplateLiterals": true}]
- radix: 1
- semi: [2, "always"]
- semi-spacing: [2, {"before": false, "after": true}]
- sort-vars: 0
- keyword-spacing: 2
- space-before-blocks: [2, "always"]
- space-before-function-paren: [2, "never"]
- space-in-parens: 0
- space-infix-ops: 2
- space-unary-ops: [2, { "words": true, "nonwords": false }]
- spaced-comment: [1, "always"]
- use-isnan: 2
- valid-jsdoc: 0
- valid-typeof: 2
- vars-on-top: 2
- wrap-iife: 0
- wrap-regex: 0
- yoda: [2, "never"]
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/app/agentsHandler.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/agentsHandler.js b/modules/web-console/backend/app/agentsHandler.js
index 9ee64ce..eb3a1e0 100644
--- a/modules/web-console/backend/app/agentsHandler.js
+++ b/modules/web-console/backend/app/agentsHandler.js
@@ -131,6 +131,7 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
this._agentSockets = new AgentSockets();
this.clusters = [];
+ this.topLsnrs = [];
}
/**
@@ -212,6 +213,15 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
}
/**
+ * Add topology listener.
+ *
+ * @param lsnr
+ */
+ addTopologyListener(lsnr) {
+ this.topLsnrs.push(lsnr);
+ }
+
+ /**
* Link agent with browsers by account.
*
* @param {Socket} sock
@@ -234,7 +244,9 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
sock.on('cluster:topology', (top) => {
const cluster = this.getOrCreateCluster(top);
- if (_.isNil(agentSocket.cluster)) {
+ _.forEach(this.topLsnrs, (lsnr) => lsnr(agentSocket, cluster, top));
+
+ if (agentSocket.cluster !== cluster) {
agentSocket.cluster = cluster;
_.forEach(tokens, (token) => {
@@ -254,11 +266,11 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
}
});
- sock.on('cluster:collector', (top) => {
+ sock.on('cluster:disconnected', () => {
+ const newTop = _.assign({}, agentSocket.cluster, {nids: []});
- });
+ _.forEach(this.topLsnrs, (lsnr) => lsnr(agentSocket, agentSocket.cluster, newTop));
- sock.on('cluster:disconnected', () => {
agentSocket.cluster = null;
_.forEach(tokens, (token) => {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/app/browsersHandler.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index 198bc12..a8ca706 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -133,6 +133,10 @@ module.exports = {
_.forEach(socks, (sock) => sock.emit('cluster:changed', cluster));
}
+ pushInitialData(sock) {
+ // Send initial data.
+ }
+
emitNotification(sock) {
sock.emit('user:notifications', this.notification);
}
@@ -185,7 +189,8 @@ module.exports = {
* @return {Promise.<T>}
*/
executeOnNode(agent, demo, params) {
- return agent.then((agentSock) => agentSock.emitEvent('node:rest', {uri: 'ignite', demo, params}))
+ return agent
+ .then((agentSock) => agentSock.emitEvent('node:rest', {uri: 'ignite', demo, params}))
.then((res) => {
if (res.status === 0) {
if (res.zipped)
@@ -310,6 +315,7 @@ module.exports = {
this.agentListeners(sock);
this.nodeListeners(sock);
+ this.pushInitialData(sock);
this.agentStats(sock.request.user.token, [sock]);
this.emitNotification(sock);
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/app/mongo.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index de91676..955cd47 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -41,8 +41,10 @@ const defineSchema = (mongoose) => {
firstName: String,
lastName: String,
email: String,
+ phone: String,
company: String,
country: String,
+ registered: Date,
lastLogin: Date,
lastActivity: Date,
admin: Boolean,
@@ -60,12 +62,14 @@ const defineSchema = (mongoose) => {
return {
_id: ret._id,
email: ret.email,
+ phone: ret.phone,
firstName: ret.firstName,
lastName: ret.lastName,
company: ret.company,
country: ret.country,
admin: ret.admin,
token: ret.token,
+ registered: ret.registered,
lastLogin: ret.lastLogin,
lastActivity: ret.lastActivity
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/config/settings.json.sample
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/config/settings.json.sample b/modules/web-console/backend/config/settings.json.sample
index 71a64ea..aa93e07 100644
--- a/modules/web-console/backend/config/settings.json.sample
+++ b/modules/web-console/backend/config/settings.json.sample
@@ -12,11 +12,11 @@
},
"mail": {
"service": "",
- "sign": "Kind regards,<br>Apache Ignite Team",
- "greeting": "Apache Ignite Web Console",
- "from": "Apache Ignite Web Console <so...@somecompany.tld>",
+ "from": "Some Company Web Console <so...@some_company.com>",
+ "greeting": "Some Company Web Console",
+ "sign": "Kind regards,<br>Some Company Team",
"auth": {
- "user": "someusername@somecompany.tld",
+ "user": "some_username@some_company.com",
"pass": ""
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/migrations/1508395969410-init-registered-date.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/migrations/1508395969410-init-registered-date.js b/modules/web-console/backend/migrations/1508395969410-init-registered-date.js
new file mode 100644
index 0000000..b994cac
--- /dev/null
+++ b/modules/web-console/backend/migrations/1508395969410-init-registered-date.js
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const _ = require('lodash');
+
+exports.up = function up(done) {
+ const accounts = this('Account');
+
+ accounts.find({}).lean().exec()
+ .then((data) => _.forEach(data, (acc) => accounts.update({_id: acc._id}, {$set: {registered: acc.lastLogin}}, {upsert: true}).exec()))
+ .then(() => done())
+ .catch(done);
+};
+
+exports.down = function down(done) {
+ this('Account').update({}, {$unset: {registered: 1}}, {multi: true}).exec()
+ .then(() => done())
+ .catch(done);
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/package.json
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/package.json b/modules/web-console/backend/package.json
index ba442f9..889c40d 100644
--- a/modules/web-console/backend/package.json
+++ b/modules/web-console/backend/package.json
@@ -49,6 +49,7 @@
"dependencies": {
"app-module-path": "2.2.0",
"body-parser": "1.17.2",
+ "common-tags": "1.4.0",
"connect-mongo": "1.3.2",
"cookie-parser": "1.4.3",
"express": "4.15.3",
@@ -62,6 +63,7 @@
"mongodb-prebuilt": "6.3.3",
"mongoose": "4.11.4",
"morgan": "1.8.2",
+ "nexmo": "2.0.2",
"nconf": "0.8.4",
"nodemailer": "4.0.1",
"passport": "0.3.2",
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/Utils.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/Utils.js b/modules/web-console/backend/services/Utils.js
new file mode 100644
index 0000000..ec97e4e
--- /dev/null
+++ b/modules/web-console/backend/services/Utils.js
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+// Fire me up!
+
+module.exports = {
+ implements: 'services/utils'
+};
+
+/**
+ * @returns {UtilsService}
+ */
+module.exports.factory = () => {
+ class UtilsService {
+ /**
+ * Generate token string.
+ *
+ * @param len length of string
+ * @returns {String} Random string.
+ */
+ static randomString(len) {
+ const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
+ const possibleLen = possible.length;
+
+ let res = '';
+
+ for (let i = 0; i < len; i++)
+ res += possible.charAt(Math.floor(Math.random() * possibleLen));
+
+ return res;
+ }
+ }
+
+ return UtilsService;
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/auth.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/auth.js b/modules/web-console/backend/services/auth.js
index dde2460..986ed95 100644
--- a/modules/web-console/backend/services/auth.js
+++ b/modules/web-console/backend/services/auth.js
@@ -21,38 +21,20 @@
module.exports = {
implements: 'services/auth',
- inject: ['mongo', 'settings', 'errors']
+ inject: ['mongo', 'settings', 'errors', 'services/utils']
};
/**
* @param mongo
* @param settings
* @param errors
+ * @param {UtilsService} utilsService
* @returns {AuthService}
*/
-module.exports.factory = (mongo, settings, errors) => {
+module.exports.factory = (mongo, settings, errors, utilsService) => {
class AuthService {
/**
- * Generate token string.
- *
- * @param length - length of string
- * @returns {string} - generated token
- */
- static generateResetToken(length) {
- length = length || settings.tokenLength;
- const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
- const possibleLen = possible.length;
-
- let res = '';
-
- for (let i = 0; i < length; i++)
- res += possible.charAt(Math.floor(Math.random() * possibleLen));
-
- return res;
- }
-
- /**
* Reset password reset token for user.
*
* @param email - user email
@@ -64,7 +46,7 @@ module.exports.factory = (mongo, settings, errors) => {
if (!user)
throw new errors.MissingResourceException('Account with that email address does not exists!');
- user.resetPasswordToken = AuthService.generateResetToken(settings.tokenLength);
+ user.resetPasswordToken = utilsService.randomString(settings.tokenLength);
return user.save();
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/caches.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/caches.js b/modules/web-console/backend/services/caches.js
index 5c96ccd..9cb65a1 100644
--- a/modules/web-console/backend/services/caches.js
+++ b/modules/web-console/backend/services/caches.js
@@ -58,6 +58,8 @@ module.exports.factory = (mongo, spaceService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_UPDATE_ERROR || err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Cache with name: "' + cache.name + '" already exist.');
+ else
+ throw err;
});
};
@@ -77,6 +79,8 @@ module.exports.factory = (mongo, spaceService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Cache with name: "' + cache.name + '" already exist.');
+ else
+ throw err;
});
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/clusters.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/clusters.js b/modules/web-console/backend/services/clusters.js
index 9f50ede..49e6f09 100644
--- a/modules/web-console/backend/services/clusters.js
+++ b/modules/web-console/backend/services/clusters.js
@@ -58,6 +58,8 @@ module.exports.factory = (mongo, spacesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_UPDATE_ERROR || err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Cluster with name: "' + cluster.name + '" already exist.');
+ else
+ throw err;
});
};
@@ -77,6 +79,8 @@ module.exports.factory = (mongo, spacesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Cluster with name: "' + cluster.name + '" already exist.');
+ else
+ throw err;
});
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/domains.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/domains.js b/modules/web-console/backend/services/domains.js
index e25d091..986991d 100644
--- a/modules/web-console/backend/services/domains.js
+++ b/modules/web-console/backend/services/domains.js
@@ -65,6 +65,8 @@ module.exports.factory = (mongo, spacesService, cachesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_UPDATE_ERROR || err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Domain model with value type: "' + domain.valueType + '" already exist.');
+ else
+ throw err;
});
};
@@ -86,6 +88,8 @@ module.exports.factory = (mongo, spacesService, cachesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Domain model with value type: "' + domain.valueType + '" already exist.');
+ else
+ throw err;
});
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/igfss.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/igfss.js b/modules/web-console/backend/services/igfss.js
index e9ff38e..5296f16 100644
--- a/modules/web-console/backend/services/igfss.js
+++ b/modules/web-console/backend/services/igfss.js
@@ -56,6 +56,8 @@ module.exports.factory = (mongo, spacesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_UPDATE_ERROR || err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('IGFS with name: "' + igfs.name + '" already exist.');
+ else
+ throw err;
});
};
@@ -74,6 +76,8 @@ module.exports.factory = (mongo, spacesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('IGFS with name: "' + igfs.name + '" already exist.');
+ else
+ throw err;
});
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/mails.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/mails.js b/modules/web-console/backend/services/mails.js
index 6a31e93..c16db3b 100644
--- a/modules/web-console/backend/services/mails.js
+++ b/modules/web-console/backend/services/mails.js
@@ -32,44 +32,44 @@ module.exports = {
* @returns {MailsService}
*/
module.exports.factory = (settings) => {
- /**
- * Send mail to user.
- *
- * @param {Account} user
- * @param {String} subject
- * @param {String} html
- * @param {String} sendErr
- * @throws {Error}
- * @return {Promise}
- */
- const send = (user, subject, html, sendErr) => {
- return new Promise((resolve, reject) => {
- const transportConfig = settings.mail;
-
- if (_.isEmpty(transportConfig.service) || _.isEmpty(transportConfig.auth.user) || _.isEmpty(transportConfig.auth.pass))
- throw new Error('Failed to send email. SMTP server is not configured. Please ask webmaster to setup SMTP server!');
-
- const mailer = nodemailer.createTransport(transportConfig);
-
- const sign = settings.mail.sign ? `<br><br>--------------<br>${settings.mail.sign}<br>` : '';
-
- const mail = {
- from: settings.mail.from,
- to: settings.mail.address(`${user.firstName} ${user.lastName}`, user.email),
- subject,
- html: html + sign
- };
-
- mailer.sendMail(mail, (err) => {
- if (err)
- return reject(sendErr ? new Error(sendErr) : err);
-
- resolve(user);
+ class MailsService {
+ /**
+ * Send mail to user.
+ *
+ * @param {Account} user
+ * @param {String} subject
+ * @param {String} html
+ * @param {String} sendErr
+ * @throws {Error}
+ * @return {Promise}
+ */
+ static send(user, subject, html, sendErr) {
+ return new Promise((resolve, reject) => {
+ const transportConfig = settings.mail;
+
+ if (_.isEmpty(transportConfig.service) || _.isEmpty(transportConfig.auth.user) || _.isEmpty(transportConfig.auth.pass))
+ throw new Error('Failed to send email. SMTP server is not configured. Please ask webmaster to setup SMTP server!');
+
+ const mailer = nodemailer.createTransport(transportConfig);
+
+ const sign = settings.mail.sign ? `<br><br>--------------<br>${settings.mail.sign}<br>` : '';
+
+ const mail = {
+ from: settings.mail.from,
+ to: settings.mail.address(`${user.firstName} ${user.lastName}`, user.email),
+ subject,
+ html: html + sign
+ };
+
+ mailer.sendMail(mail, (err) => {
+ if (err)
+ return reject(sendErr ? new Error(sendErr) : err);
+
+ resolve(user);
+ });
});
- });
- };
+ }
- class MailsService {
/**
* Send email to user for password reset.
* @param host
@@ -78,7 +78,7 @@ module.exports.factory = (settings) => {
static emailUserSignUp(host, user) {
const resetLink = `${host}/password/reset?token=${user.resetPasswordToken}`;
- return send(user, `Thanks for signing up for ${settings.mail.greeting}.`,
+ return MailsService.send(user, `Thanks for signing up for ${settings.mail.greeting}.`,
`Hello ${user.firstName} ${user.lastName}!<br><br>` +
`You are receiving this email because you have signed up to use <a href="${host}">${settings.mail.greeting}</a>.<br><br>` +
'If you have not done the sign up and do not know what this email is about, please ignore it.<br>' +
@@ -94,7 +94,7 @@ module.exports.factory = (settings) => {
static emailUserResetLink(host, user) {
const resetLink = `${host}/password/reset?token=${user.resetPasswordToken}`;
- return send(user, 'Password Reset',
+ return MailsService.send(user, 'Password Reset',
`Hello ${user.firstName} ${user.lastName}!<br><br>` +
'You are receiving this because you (or someone else) have requested the reset of the password for your account.<br><br>' +
'Please click on the following link, or paste this into your browser to complete the process:<br><br>' +
@@ -109,7 +109,7 @@ module.exports.factory = (settings) => {
* @param user
*/
static emailPasswordChanged(host, user) {
- return send(user, 'Your password has been changed',
+ return MailsService.send(user, 'Your password has been changed',
`Hello ${user.firstName} ${user.lastName}!<br><br>` +
`This is a confirmation that the password for your account on <a href="${host}">${settings.mail.greeting}</a> has just been changed.<br><br>`,
'Password was changed, but failed to send confirmation email!');
@@ -121,7 +121,7 @@ module.exports.factory = (settings) => {
* @param user
*/
static emailUserDeletion(host, user) {
- return send(user, 'Your account was removed',
+ return MailsService.send(user, 'Your account was removed',
`Hello ${user.firstName} ${user.lastName}!<br><br>` +
`You are receiving this email because your account for <a href="${host}">${settings.mail.greeting}</a> was removed.`,
'Account was removed, but failed to send email notification to user!');
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/notebooks.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/notebooks.js b/modules/web-console/backend/services/notebooks.js
index 5d8b57d..4ae89d2 100644
--- a/modules/web-console/backend/services/notebooks.js
+++ b/modules/web-console/backend/services/notebooks.js
@@ -51,6 +51,8 @@ module.exports.factory = (mongo, spacesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_UPDATE_ERROR || err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Notebook with name: "' + notebook.name + '" already exist.');
+ else
+ throw err;
});
};
@@ -65,6 +67,8 @@ module.exports.factory = (mongo, spacesService, errors) => {
.catch((err) => {
if (err.code === mongo.errCodes.DUPLICATE_KEY_ERROR)
throw new errors.DuplicateKeyException('Notebook with name: "' + notebook.name + '" already exist.');
+ else
+ throw err;
});
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/backend/services/users.js
----------------------------------------------------------------------
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 620f7e9..f804840 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -23,7 +23,7 @@ const _ = require('lodash');
module.exports = {
implements: 'services/users',
- inject: ['errors', 'settings', 'mongo', 'services/spaces', 'services/mails', 'services/activities', 'agents-handler']
+ inject: ['errors', 'settings', 'mongo', 'services/spaces', 'services/mails', 'services/activities', 'services/utils', 'agents-handler']
};
/**
@@ -33,22 +33,11 @@ module.exports = {
* @param {SpacesService} spacesService
* @param {MailsService} mailsService
* @param {ActivitiesService} activitiesService
+ * @param {UtilsService} utilsService
* @param {AgentsHandler} agentHnd
* @returns {UsersService}
*/
-module.exports.factory = (errors, settings, mongo, spacesService, mailsService, activitiesService, agentHnd) => {
- const _randomString = () => {
- const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
- const possibleLen = possible.length;
-
- let res = '';
-
- for (let i = 0; i < settings.tokenLength; i++)
- res += possible.charAt(Math.floor(Math.random() * possibleLen));
-
- return res;
- };
-
+module.exports.factory = (errors, settings, mongo, spacesService, mailsService, activitiesService, utilsService, agentHnd) => {
class UsersService {
/**
* Save profile information.
@@ -61,8 +50,8 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService,
return mongo.Account.count().exec()
.then((cnt) => {
user.admin = cnt === 0;
-
- user.token = _randomString();
+ user.registered = new Date();
+ user.token = utilsService.randomString(settings.tokenLength);
return new mongo.Account(user);
})
@@ -80,7 +69,7 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService,
});
})
.then((registered) => {
- registered.resetPasswordToken = _randomString();
+ registered.resetPasswordToken = utilsService.randomString(settings.tokenLength);
return registered.save()
.then(() => mongo.Space.create({name: 'Personal space', owner: registered._id}))
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/app.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 5a27ea2..a1cd6eb 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -124,9 +124,12 @@ import pageConfigureAdvanced from './components/page-configure-advanced';
import pageQueries from './components/page-queries';
import gridColumnSelector from './components/grid-column-selector';
import gridItemSelected from './components/grid-item-selected';
+import gridNoData from './components/grid-no-data';
+import gridExport from './components/grid-export';
import bsSelectMenu from './components/bs-select-menu';
import protectFromBsSelectRender from './components/protect-from-bs-select-render';
import uiGridHovering from './components/ui-grid-hovering';
+import uiGridFilters from './components/ui-grid-filters';
import listEditable from './components/list-editable';
import clusterSelector from './components/cluster-selector';
import connectedClusters from './components/connected-clusters';
@@ -201,8 +204,11 @@ angular.module('ignite-console', [
pageQueries.name,
gridColumnSelector.name,
gridItemSelected.name,
+ gridNoData.name,
+ gridExport.name,
bsSelectMenu.name,
uiGridHovering.name,
+ uiGridFilters.name,
protectFromBsSelectRender.name,
AngularStrapTooltip.name,
AngularStrapSelect.name,
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/bs-select-menu/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/style.scss b/modules/web-console/frontend/app/components/bs-select-menu/style.scss
index ccf33a3..d82bf19 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/style.scss
+++ b/modules/web-console/frontend/app/components/bs-select-menu/style.scss
@@ -31,7 +31,6 @@
overflow-y: auto;
overflow-x: hidden;
max-height: $max-visible-items * $item-height;
- max-width: 280px;
box-shadow: 0 2px 5px 0 rgba(0, 0, 0, 0.3);
border-radius: $ignite-button-border-radius;
border: 1px solid #c5c5c5;
@@ -76,6 +75,15 @@
}
}
+ .bssm-click-overlay {
+ position: fixed;
+ top: 0;
+ right: 0;
+ bottom: 0;
+ left: 0;
+ z-index: -1;
+ }
+
&.bssm-multiple {
.bssm-active-indicator {
display: initial;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/bs-select-menu/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/template.pug b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
index a9c1c28..c8c6eaa 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/template.pug
+++ b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
@@ -25,11 +25,8 @@ ul.bs-select-menu(
ng-click='$ctrl.areAllSelected() ? $selectNone() : $selectAll()'
type='button'
)
- span.bssm-active-indicator.icon.icon-left.fa(
- ng-class=`{
- 'fa-check-square bssm-active-indicator__active': $ctrl.areAllSelected(),
- 'fa-square-o': !$ctrl.areAllSelected()
- }`
+ img.bssm-active-indicator.icon-left(
+ ng-src='{{ $ctrl.areAllSelected() ? "/images/checkbox-active.svg" : "/images/checkbox.svg" }}'
)
| All
li(role='presentation' ng-repeat='match in $matches')
@@ -42,10 +39,8 @@ ul.bs-select-menu(
data-placement='right auto'
title='{{ ::match.label }}'
)
- span.bssm-active-indicator.icon.icon-left.fa(
- ng-class=`{
- 'fa-check-square bssm-active-indicator__active': $isActive($index),
- 'fa-square-o': !$isActive($index)
- }`
+ img.bssm-active-indicator.icon-left(
+ ng-src='{{ $isActive($index) ? "/images/checkbox-active.svg" : "/images/checkbox.svg" }}'
)
span.bssm-item-text(ng-bind-html='match.label')
+ .bssm-click-overlay(ng-click='$hide()')
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-export/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-export/component.js b/modules/web-console/frontend/app/components/grid-export/component.js
new file mode 100644
index 0000000..9c239a6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-export/component.js
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+
+export default {
+ template,
+ controller: class {
+ static $inject = ['$scope', 'uiGridGroupingConstants', 'uiGridExporterService', 'uiGridExporterConstants'];
+
+ constructor($scope, uiGridGroupingConstants, uiGridExporterService, uiGridExporterConstants) {
+ Object.assign(this, { uiGridGroupingConstants, uiGridExporterService, uiGridExporterConstants });
+ }
+
+ export() {
+ const data = [];
+ const columnHeaders = this.uiGridExporterService.getColumnHeaders(this.gridApi.grid, this.uiGridExporterConstants.VISIBLE);
+
+ _.forEach(this.gridApi.grid.rows, (row) => {
+ if (!row.visible)
+ return;
+
+ const values = [];
+ _.forEach(columnHeaders, ({ name }) => {
+ values.push({ value: row.entity[name] });
+ });
+
+ data.push(values);
+ });
+
+ const csvContent = this.uiGridExporterService.formatAsCsv(columnHeaders, data, this.gridApi.grid.options.exporterCsvColumnSeparator);
+ this.uiGridExporterService.downloadFile(this.gridApi.grid.options.exporterCsvFilename, csvContent, this.gridApi.grid.options.exporterOlderExcelCompatibility);
+ }
+ },
+ bindings: {
+ gridApi: '<'
+ }
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-export/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-export/index.js b/modules/web-console/frontend/app/components/grid-export/index.js
new file mode 100644
index 0000000..9fa8835
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-export/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import component from './component';
+
+export default angular
+ .module('ignite-console.grid-export', [])
+ .component('gridExport', component);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-export/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-export/template.pug b/modules/web-console/frontend/app/components/grid-export/template.pug
new file mode 100644
index 0000000..99780ee
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-export/template.pug
@@ -0,0 +1,18 @@
+//-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+button.btn-ignite.btn-ignite--primary-outline(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
+ svg(ignite-icon='csv')
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-no-data/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-no-data/component.js b/modules/web-console/frontend/app/components/grid-no-data/component.js
new file mode 100644
index 0000000..aa219df
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-no-data/component.js
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './style.scss';
+import controller from './controller.js';
+
+export default {
+ template: `
+ <div ng-if='$ctrl.noData' ng-transclude></div>
+ <div ng-if='$ctrl.noDataFiltered' ng-transclude='noDataFiltered'></div>
+ `,
+ controller,
+ bindings: {
+ gridApi: '<'
+ },
+ transclude: {
+ noDataFiltered: '?gridNoDataFiltered'
+ }
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-no-data/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-no-data/controller.js b/modules/web-console/frontend/app/components/grid-no-data/controller.js
new file mode 100644
index 0000000..95e8a5e
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-no-data/controller.js
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import filter from 'lodash/fp/filter';
+
+const rowsFiltered = filter(({ visible }) => visible);
+
+export default class {
+ static $inject = ['$scope', 'uiGridConstants'];
+
+ constructor($scope, uiGridConstants) {
+ Object.assign(this, {$scope, uiGridConstants});
+
+ this.noData = true;
+ }
+
+ $onChanges(changes) {
+ if (changes && 'gridApi' in changes && changes.gridApi.currentValue) {
+ this.gridApi.core.on.rowsVisibleChanged(this.$scope, () => {
+ this.applyValues();
+ });
+ }
+ }
+
+ applyValues() {
+ if (!this.gridApi.grid.rows.length) {
+ this.noData = true;
+ return;
+ }
+
+ this.noData = false;
+
+ const filtered = rowsFiltered(this.gridApi.grid.rows);
+ this.noDataFiltered = !filtered.length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-no-data/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-no-data/index.js b/modules/web-console/frontend/app/components/grid-no-data/index.js
new file mode 100644
index 0000000..2acecf9
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-no-data/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import component from './component';
+
+export default angular
+ .module('ignite-console.grid-no-data', [])
+ .component('gridNoData', component);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/grid-no-data/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/grid-no-data/style.scss b/modules/web-console/frontend/app/components/grid-no-data/style.scss
new file mode 100644
index 0000000..0a51ac2
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-no-data/style.scss
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grid-no-data {
+ position: relative;
+ display: block;
+ padding: 0 51px;
+
+ border-radius: 0 0 4px 4px;
+
+ font-style: italic;
+ line-height: 16px;
+
+ [ng-transclude] {
+ padding: 16px 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
index 2331173..f160707 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
@@ -15,7 +15,6 @@
limitations under the License.
.list-editable-cols__header(
- ng-show='$ctrl.ngModel.$viewValue.length'
ng-class='::$ctrl.rowClass'
)
.list-editable-cols__header-cell(ng-repeat='col in ::$ctrl.colDefs' ng-class='::col.cellClass')
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js
index e427ab5..32d75f9 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/row.directive.js
@@ -33,7 +33,7 @@ export default function() {
el.addClass(ctrl.rowClass);
ctrl.colDefs.forEach(({ cellClass }, index) => {
- children[index].classList.add(...(Array.isArray(cellClass) ? cellClass : [cellClass]));
+ _.forEach((Array.isArray(cellClass) ? cellClass : [cellClass]), (item) => children[index].classList.add(item));
});
}
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/list-editable/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/style.scss b/modules/web-console/frontend/app/components/list-editable/style.scss
index 0f3f8ae..83ce0d4 100644
--- a/modules/web-console/frontend/app/components/list-editable/style.scss
+++ b/modules/web-console/frontend/app/components/list-editable/style.scss
@@ -33,7 +33,8 @@ list-editable {
display: flex;
align-items: center;
min-height: $min-height;
- padding: 5px 10px;
+ padding: 5px 20px;
+ margin: -6px 0;
font-style: italic;
}
@@ -76,6 +77,10 @@ list-editable {
justify-content: center;
}
+ &-sort {
+ display: none;
+ }
+
&-cross {
[ignite-icon] {
width: 12px;
@@ -105,11 +110,6 @@ list-editable {
}
&:not(.le-row--has-item-view) {
- & > .le-row-index,
- & > .le-row-cross {
- margin-top: 18px;
- }
-
align-items: flex-start;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/list-editable/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-editable/template.pug b/modules/web-console/frontend/app/components/list-editable/template.pug
index a713188..2900602 100644
--- a/modules/web-console/frontend/app/components/list-editable/template.pug
+++ b/modules/web-console/frontend/app/components/list-editable/template.pug
@@ -14,8 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-.le-body(ng-if='$ctrl.ngModel.$viewValue.length')
+.le-body()
.le-row(
+ ng-if='$ctrl.ngModel.$viewValue.length'
ng-repeat='item in $ctrl.ngModel.$viewValue track by $ctrl.$index(item, $index)'
ng-class=`{
'le-row--editable': $ctrl.isEditView($index),
@@ -46,4 +47,5 @@
button.btn-ignite.btn-ignite--link-dashed-secondary(type='button' ng-click='$ctrl.remove($index)')
svg(ignite-icon='cross')
-div(ng-transclude='noItems' ng-if='!$ctrl.ngModel.$viewValue.length')
+ .le-row(ng-if='!$ctrl.ngModel.$viewValue.length')
+ div(ng-transclude='noItems')
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
index e311246..11ff7bc 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/list-of-registered-users.tpl.pug
@@ -45,7 +45,7 @@ include /app/helpers/jade/mixins
required: false,
options: '$ctrl.actionOptions'
})
- button.btn-ignite.btn-ignite--primary-outline(ng-click='$ctrl.exportCsv()' bs-tooltip='' data-title='Export table to csv' data-placement='top')
+ button.btn-ignite.btn-ignite--primary-outline(ng-click='$ctrl.exportCsv()' bs-tooltip='' data-title='Export table to CSV' data-placement='top')
svg(ignite-icon='csv')
form.ui-grid-settings-dateperiod(name=form novalidate)
-var form = 'admin'
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
new file mode 100644
index 0000000..2e18933
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import './style.scss';
+
+export default function uiGridFilters(uiGridConstants) {
+ return {
+ require: 'uiGrid',
+ link: {
+ pre(scope, el, attr, grid) {
+ if (!grid.grid.options.enableFiltering) return;
+ grid.grid.options.columnDefs.filter((cd) => cd.multiselectFilterOptions).forEach((cd) => {
+ cd.headerCellTemplate = template;
+ cd.filter = {
+ type: uiGridConstants.filter.SELECT,
+ term: cd.multiselectFilterOptions.map((t) => t.value),
+ condition(searchTerm, cellValue) {
+ return searchTerm.includes(cellValue);
+ },
+ selectOptions: cd.multiselectFilterOptions,
+ $$selectOptionsMapping: cd.multiselectFilterOptions.reduce((a, v) => Object.assign(a, {[v.value]: v.label}), {}),
+ $$multiselectFilterTooltip() {
+ const prefix = 'Active filter';
+ switch (this.term.length) {
+ case 0:
+ return `${prefix}: show none`;
+ default:
+ return `${prefix}: ${this.term.map((t) => this.$$selectOptionsMapping[t]).join(', ')}`;
+ case this.selectOptions.length:
+ return `${prefix}: show all`;
+ }
+ }
+ };
+ if (!cd.cellTemplate) {
+ cd.cellTemplate = `
+ <div class="ui-grid-cell-contents">
+ {{ col.colDef.filter.$$selectOptionsMapping[row.entity[col.field]] }}
+ </div>
+ `;
+ }
+ });
+ }
+ }
+ };
+}
+
+uiGridFilters.$inject = ['uiGridConstants'];
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/ui-grid-filters/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/index.js b/modules/web-console/frontend/app/components/ui-grid-filters/index.js
new file mode 100644
index 0000000..0f05b77
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/index.js
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import directive from './directive';
+import flow from 'lodash/flow';
+
+export default angular
+ .module('ignite-console.ui-grid-filters', ['ui.grid'])
+ .decorator('$tooltip', ['$delegate', ($delegate) => {
+ return function(el, config) {
+ const instance = $delegate(el, config);
+ instance.$referenceElement = el;
+ instance.destroy = flow(instance.destroy, () => instance.$referenceElement = null);
+ instance.$applyPlacement = flow(instance.$applyPlacement, () => {
+ if (!instance.$element) return;
+ const refWidth = instance.$referenceElement[0].getBoundingClientRect().width;
+ const elWidth = instance.$element[0].getBoundingClientRect().width;
+ if (refWidth > elWidth) {
+ instance.$element.css({
+ width: refWidth,
+ maxWidth: 'initial'
+ });
+ }
+ });
+ return instance;
+ };
+ }])
+ .directive('uiGridFilters', directive);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/ui-grid-filters/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/style.scss b/modules/web-console/frontend/app/components/ui-grid-filters/style.scss
new file mode 100644
index 0000000..629cbad
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/style.scss
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.ui-grid-filters[role="columnheader"] {
+ display: flex;
+
+ // Decrease horizontal padding because multiselect button already has it
+ padding-left: 8px !important;
+ padding-right: 8px !important;
+
+ & > div:first-child {
+ flex: auto !important;
+ }
+
+ .ui-grid-cell-contents[role="button"] {
+ flex: auto !important;
+ flex-basis: 100% !important;
+
+ padding: 0 !important;
+ overflow: visible !important;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/ui-grid-filters/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/template.pug b/modules/web-console/frontend/app/components/ui-grid-filters/template.pug
new file mode 100644
index 0000000..c898078
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/template.pug
@@ -0,0 +1,47 @@
+//-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+.ui-grid-filter-container.ui-grid-filters(role='columnheader')
+ div(ng-style='col.extraStyle'
+ ng-repeat='colFilter in col.filters'
+ ng-class="{'ui-grid-filter-cancel-button-hidden' : colFilter.disableCancelFilterButton === true }"
+ ng-switch='colFilter.type')
+ div(ng-switch-when='select')
+ button.btn-ignite.btn-ignite--link-dashed-success(
+ ng-class=`{
+ 'bold': colFilter.term.length !== colFilter.selectOptions.length
+ }`
+ type='button'
+ title='{{ colFilter.$$multiselectFilterTooltip() }}'
+ ng-model='colFilter.term'
+ bs-select
+ bs-options='option.value as option.label for option in colFilter.selectOptions'
+ data-multiple='true'
+ data-trigger='click'
+ data-placement='bottom-left'
+ protect-from-bs-select-render
+ ) {{ col.displayName }}
+
+ .ui-grid-cell-contents(role='button')
+ button.btn-ignite.btn-ignite--link-dashed-success(
+ ui-grid-one-bind-id-grid="col.uid + '-sortdir-text'"
+ ui-grid-visible="col.sort.direction"
+ aria-label="Sort Descending")
+ i(ng-class="{\
+ 'ui-grid-icon-up-dir': col.sort.direction == 'asc',\
+ 'ui-grid-icon-down-dir': col.sort.direction == 'desc',\
+ 'ui-grid-icon-blank': !col.sort.direction\
+ }" title="" aria-hidden="true")
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/component.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/component.js b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/component.js
new file mode 100644
index 0000000..1621adc
--- /dev/null
+++ b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/component.js
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+
+export default {
+ template
+};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug
new file mode 100644
index 0000000..0545db1
--- /dev/null
+++ b/modules/web-console/frontend/app/components/web-console-header/components/web-console-header-extension/template.pug
@@ -0,0 +1,15 @@
+//-
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/web-console-header/index.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/web-console-header/index.js b/modules/web-console/frontend/app/components/web-console-header/index.js
index e41e1cc..3fc5c66 100644
--- a/modules/web-console/frontend/app/components/web-console-header/index.js
+++ b/modules/web-console/frontend/app/components/web-console-header/index.js
@@ -17,7 +17,9 @@
import angular from 'angular';
import component from './component';
+import componentExtension from './components/web-console-header-extension/component';
export default angular
.module('ignite-console.web-console-header', [])
- .component('webConsoleHeader', component);
+ .component('webConsoleHeader', component)
+ .component('webConsoleHeaderExtension', componentExtension);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/web-console-header/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/web-console-header/style.scss b/modules/web-console/frontend/app/components/web-console-header/style.scss
index f221a00..db49c3e 100644
--- a/modules/web-console/frontend/app/components/web-console-header/style.scss
+++ b/modules/web-console/frontend/app/components/web-console-header/style.scss
@@ -18,7 +18,7 @@
web-console-header {
@import "./../../../public/stylesheets/variables.scss";
- $nav-item-margin: 40px;
+ $nav-item-margin: 30px;
$bottom-border-width: 4px;
display: block;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/components/web-console-header/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/web-console-header/template.pug b/modules/web-console/frontend/app/components/web-console-header/template.pug
index 41586b7..660874e 100644
--- a/modules/web-console/frontend/app/components/web-console-header/template.pug
+++ b/modules/web-console/frontend/app/components/web-console-header/template.pug
@@ -16,10 +16,10 @@
.wch-notification(ng-show='$ctrl.UserNotifications.message' ng-bind-html='$ctrl.UserNotifications.message')
-.wch-notification(ng-show='$ctrl.$rootScope.user.becomeUsed')
- | You are currently viewing user #[strong {{$ctrl.$rootScope.user.firstName}} {{$ctrl.$rootScope.user.lastName}}] as administrator. #[a(ng-click='$ctrl.$rootScope.revertIdentity()') Revert to your identity?]
+.wch-notification(ng-show='$root.user.becomeUsed')
+ | You are currently viewing user #[strong {{$root.user.firstName}} {{$root.user.lastName}}] as administrator. #[a(ng-click='$root.revertIdentity()') Revert to your identity?]
-.wch-notification.wch-notification--demo(ng-if='$ctrl.$rootScope.IgniteDemoMode')
+.wch-notification.wch-notification--demo(ng-if='$root.IgniteDemoMode')
.container(ng-controller='demoController')
| You are now in #[b Demo Mode]. #[a(ng-click='closeDemo();') Close Demo?]
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/modules/form/field/input/text.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/form/field/input/text.scss b/modules/web-console/frontend/app/modules/form/field/input/text.scss
index 6882469..658d740 100644
--- a/modules/web-console/frontend/app/modules/form/field/input/text.scss
+++ b/modules/web-console/frontend/app/modules/form/field/input/text.scss
@@ -17,6 +17,7 @@
.checkbox label .input-tip {
position: initial;
+ overflow: visible;
}
.input-tip .fa-floppy-o {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/modules/user/permissions.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/user/permissions.js b/modules/web-console/frontend/app/modules/user/permissions.js
index b6f7c3a..616226a 100644
--- a/modules/web-console/frontend/app/modules/user/permissions.js
+++ b/modules/web-console/frontend/app/modules/user/permissions.js
@@ -16,8 +16,8 @@
*/
const guest = ['login'];
-const becomed = ['profile', 'configuration', 'query', 'demo'];
-const user = becomed.concat(['logout']);
+const becomed = ['profile', 'configuration'];
+const user = becomed.concat(['logout', 'query', 'demo']);
const admin = user.concat(['admin_page']);
export default {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/primitives/btn/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/btn/index.scss b/modules/web-console/frontend/app/primitives/btn/index.scss
index 94b5bd2..2870cff 100644
--- a/modules/web-console/frontend/app/primitives/btn/index.scss
+++ b/modules/web-console/frontend/app/primitives/btn/index.scss
@@ -242,6 +242,14 @@ $btn-content-padding-with-border: 9px 11px;
@include btn-ignite--link-dashed($color, $activeHover, $disabled);
}
+.btn-ignite--link-dashed-primary {
+ $color: $ignite-brand-primary;
+ $activeHover: change-color($color, $lightness: 26%);
+ $disabled: change-color($color, $saturation: 57%, $lightness: 68%);
+
+ @include btn-ignite--link-dashed($color, $activeHover, $disabled);
+}
+
.btn-ignite--link-dashed-secondary {
$activeHover: change-color($ignite-brand-success, $lightness: 26%);
@include btn-ignite--link-dashed($text-color, $activeHover, $gray-light);
@@ -319,3 +327,12 @@ $btn-content-padding-with-border: 9px 11px;
$color-hover: change-color($ignite-brand-success, $lightness: 26%)
);
}
+
+.btn-ignite--link {
+ background: transparent;
+
+ @include ignite-link(
+ $color: $ignite-brand-success,
+ $color-hover: change-color($ignite-brand-success, $lightness: 26%)
+ );
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/primitives/form-field/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss
index f6d8496..01dd941 100644
--- a/modules/web-console/frontend/app/primitives/form-field/index.scss
+++ b/modules/web-console/frontend/app/primitives/form-field/index.scss
@@ -26,7 +26,7 @@
width: auto;
}
- .ignite-form-field__label {
+ &__label {
float: left;
width: 100%;
margin: 0 10px 4px;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss b/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
index e0cf139..56bab22 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
@@ -92,6 +92,18 @@
&--heading {
cursor: default;
+
+ sub {
+ bottom: 0;
+ height: 12px;
+ margin-left: 22px;
+
+ font-family: Roboto;
+ font-size: 12px;
+ line-height: 1;
+ text-align: left;
+ color: $gray-light;
+ }
}
&--heading > span {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/app/primitives/ui-grid/index.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/primitives/ui-grid/index.scss b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
index 5caa57c..a83cb57 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
@@ -157,6 +157,7 @@
.ui-grid-filter-container {
padding-left: 20px;
padding-right: 20px;
+ font-weight: normal;
}
.ng-hide + .ui-grid-header-cell-row .ui-grid-header-cell {
@@ -519,6 +520,7 @@
}
}
+// Obsoleted, use grid-no-data.
.ui-grid--ignite.no-data {
position: relative;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/package.json
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 5b1734f..7f7671a 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -49,7 +49,7 @@
"angular-strap": "2.3.12",
"angular-translate": "2.16.0",
"angular-tree-control": "0.2.28",
- "angular-ui-grid": "4.0.7",
+ "angular-ui-grid": "4.0.11",
"babel-core": "6.25.0",
"babel-eslint": "7.2.3",
"babel-loader": "7.1.1",
@@ -62,6 +62,7 @@
"bootstrap-sass": "3.3.7",
"brace": "0.10.0",
"browser-update": "2.1.9",
+ "bson-objectid": "1.1.5",
"copy-webpack-plugin": "4.0.1",
"css-loader": "0.28.7",
"eslint": "4.3.0",
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/public/images/checkbox-active.svg
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/images/checkbox-active.svg b/modules/web-console/frontend/public/images/checkbox-active.svg
new file mode 100644
index 0000000..82e59c6
--- /dev/null
+++ b/modules/web-console/frontend/public/images/checkbox-active.svg
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="12px" height="12px" viewBox="0 0 12 12" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <!-- Generator: sketchtool 45.2 (43514) - http://www.bohemiancoding.com/sketch -->
+ <title>1F50951A-D0DF-4DE9-B464-5A57049A9426</title>
+ <desc>Created with sketchtool.</desc>
+ <defs>
+ <polygon id="path-1" points="4.0575 5.993 5.0835 7.0255 8.99916406 3.06347168 10.0556641 4.12097168 5.0885 9.1435 3 7.0505"></polygon>
+ <filter x="-28.3%" y="-16.4%" width="156.7%" height="165.8%" filterUnits="objectBoundingBox" id="filter-2">
+ <feOffset dx="0" dy="1" in="SourceAlpha" result="shadowOffsetOuter1"></feOffset>
+ <feGaussianBlur stdDeviation="0.5" in="shadowOffsetOuter1" result="shadowBlurOuter1"></feGaussianBlur>
+ <feColorMatrix values="0 0 0 0 0.029487408 0 0 0 0 0.138965552 0 0 0 0 0.210990646 0 0 0 0.5 0" type="matrix" in="shadowBlurOuter1"></feColorMatrix>
+ </filter>
+ </defs>
+ <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+ <g id="Checkbox" transform="translate(-51.000000, -254.000000)">
+ <g id="CheckboxActive" transform="translate(51.000000, 254.000000)">
+ <rect id="Rectangle-2-Copy" fill="#0065B6" x="0" y="0" width="12" height="12" rx="2"></rect>
+ <g id="Imported-Layers">
+ <use fill="black" fill-opacity="1" filter="url(#filter-2)" xlink:href="#path-1"></use>
+ <use fill="#FFFFFF" fill-rule="evenodd" xlink:href="#path-1"></use>
+ </g>
+ </g>
+ </g>
+ </g>
+</svg>
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/public/images/checkbox.svg
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/images/checkbox.svg b/modules/web-console/frontend/public/images/checkbox.svg
new file mode 100644
index 0000000..82264a9
--- /dev/null
+++ b/modules/web-console/frontend/public/images/checkbox.svg
@@ -0,0 +1,22 @@
+<svg version="1.1" viewBox="0 0 12 12" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+ <defs>
+ <linearGradient id="linearGradient-1" x1="50%" x2="50%" y1="-18.787%" y2="68.083%">
+ <stop stop-color="#979797" offset="0"/>
+ <stop stop-color="#BEBEBE" offset="1"/>
+ </linearGradient>
+ <rect id="path-2" x="148" y="254" width="12" height="12" rx="2"/>
+ <filter id="filter-3" x="-12.5%" y="-12.5%" width="125%" height="125%">
+ <feGaussianBlur in="SourceAlpha" result="shadowBlurInner1" stdDeviation="1"/>
+ <feOffset dx="0" dy="1" in="shadowBlurInner1" result="shadowOffsetInner1"/>
+ <feComposite in="shadowOffsetInner1" in2="SourceAlpha" k2="-1" k3="1" operator="arithmetic" result="shadowInnerInner1"/>
+ <feColorMatrix in="shadowInnerInner1" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0.5 0"/>
+ </filter>
+ </defs>
+ <g fill="none" fill-rule="evenodd">
+ <g id="Checkbox" transform="translate(-148 -254)">
+ <use width="100%" height="100%" fill="#ffffff" xlink:href="#path-2"/>
+ <use fill="black" filter="url(#filter-3)" xlink:href="#path-2"/>
+ <rect x="148.5" y="254.5" width="11" height="11" rx="2" stroke="url(#linearGradient-1)"/>
+ </g>
+ </g>
+</svg>
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/public/images/icons/alert.svg
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/images/icons/alert.svg b/modules/web-console/frontend/public/images/icons/alert.svg
new file mode 100644
index 0000000..6d1b3e2
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/alert.svg
@@ -0,0 +1 @@
+<svg id="alert-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 19"><path fill="currentColor" d="M8 18.7c1 0 1.8-.8 1.8-1.8H6.2c0 1 .8 1.8 1.8 1.8zM14.1 8c0-2.9-2-5.3-4.7-5.9v-.7C9.4.6 8.8 0 8 0S6.6.6 6.6 1.4V2c-2.7.7-4.7 3.1-4.7 6v5.2L0 15.1v.9h16v-.9l-1.9-1.9V8z"/></svg>
http://git-wip-us.apache.org/repos/asf/ignite/blob/e08d19cb/modules/web-console/frontend/public/images/icons/checkmark.svg
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/public/images/icons/checkmark.svg b/modules/web-console/frontend/public/images/icons/checkmark.svg
new file mode 100644
index 0000000..a7896c7
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/checkmark.svg
@@ -0,0 +1,3 @@
+<svg version="1.1" viewBox="0 0 14 11" xmlns="http://www.w3.org/2000/svg">
+<path fill="currentColor" d="m12.877 0.32617l-8.4277 8.4219-3.3184-3.3203-1.1309 1.123 4.4492 4.4492 9.5508-9.5508-1.123-1.123z"/>
+</svg>
[03/50] [abbrv] ignite git commit: IGNITE-7177 Correctly handle
custom messages which do not change affinity
Posted by sb...@apache.org.
IGNITE-7177 Correctly handle custom messages which do not change affinity
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7cf049e8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7cf049e8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7cf049e8
Branch: refs/heads/ignite-zk
Commit: 7cf049e86297427054d07298b33cc6874aa3c1c0
Parents: c10aa0c
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Tue Dec 12 19:11:46 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Tue Dec 12 19:11:46 2017 +0300
----------------------------------------------------------------------
.../cache/CacheAffinitySharedManager.java | 31 +++++++++++++++-----
.../GridDhtPartitionsExchangeFuture.java | 14 +++++++--
2 files changed, 34 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7cf049e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 8441a5e..6347d03 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -684,6 +684,28 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
}
/**
+ * @param fut Exchange future.
+ * @param crd Coordinator flag.
+ * @param exchActions Exchange actions.
+ */
+ public void onCustomMessageNoAffinityChange(
+ GridDhtPartitionsExchangeFuture fut,
+ boolean crd,
+ @Nullable final ExchangeActions exchActions
+ ) {
+ final ExchangeDiscoveryEvents evts = fut.context().events();
+
+ forAllCacheGroups(crd, new IgniteInClosureX<GridAffinityAssignmentCache>() {
+ @Override public void applyx(GridAffinityAssignmentCache aff) {
+ if (exchActions != null && exchActions.cacheGroupStopping(aff.groupId()))
+ return;
+
+ aff.clientEventTopologyChange(evts.lastEvent(), evts.topologyVersion());
+ }
+ });
+ }
+
+ /**
* Called on exchange initiated for cache start/stop request.
*
* @param fut Exchange future.
@@ -703,14 +725,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
caches.updateCachesInfo(exchActions);
// Affinity did not change for existing caches.
- forAllCacheGroups(crd, new IgniteInClosureX<GridAffinityAssignmentCache>() {
- @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
- if (exchActions.cacheGroupStopping(aff.groupId()))
- return;
-
- aff.clientEventTopologyChange(evts.lastEvent(), evts.topologyVersion());
- }
- });
+ onCustomMessageNoAffinityChange(fut, crd, exchActions);
for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
DynamicCacheDescriptor cacheDesc = action.descriptor();
http://git-wip-us.apache.org/repos/asf/ignite/blob/7cf049e8/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d29293e..a8b195d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -588,9 +588,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
exchange = onCacheChangeRequest(crdNode);
}
else if (msg instanceof SnapshotDiscoveryMessage) {
- exchange = CU.clientNode(firstDiscoEvt.eventNode()) ?
- onClientNodeEvent(crdNode) :
- onServerNodeEvent(crdNode);
+ exchange = onCustomMessageNoAffinityChange(crdNode);
}
else {
assert affChangeMsg != null : this;
@@ -894,6 +892,16 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
/**
* @param crd Coordinator flag.
+ * @return Exchange type.
+ */
+ private ExchangeType onCustomMessageNoAffinityChange(boolean crd) {
+ cctx.affinity().onCustomMessageNoAffinityChange(this, crd, exchActions);
+
+ return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
+ }
+
+ /**
+ * @param crd Coordinator flag.
* @throws IgniteCheckedException If failed.
* @return Exchange type.
*/
[42/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk-ce
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk-ce
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/246b7655
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/246b7655
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/246b7655
Branch: refs/heads/ignite-zk
Commit: 246b7655a036c792994cc7c99b1108aef4aa9c4e
Parents: 1fb53f3 8f327a1
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 19 10:57:34 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 19 10:57:34 2017 +0300
----------------------------------------------------------------------
.../processors/igfs/IgfsDataManager.java | 4 +-
...zySecondaryFileSystemPositionedReadable.java | 15 +++--
.../query/PlatformAbstractQueryCursor.java | 9 +++
.../cache/query/PlatformFieldsQueryCursor.java | 24 ++++++-
...fsSecondaryFileSystemPositionedReadable.java | 41 +++++++-----
.../ExpiryCacheHolderTest.cs | 5 ++
.../Binary/BinaryFooterTest.cs | 2 +-
.../Binary/Serializable/SqlDmlTest.cs | 6 +-
.../Cache/CacheTestAsyncWrapper.cs | 7 ++
.../Cache/Query/CacheDmlQueriesTest.cs | 18 ++---
.../Query/CacheQueriesCodeConfigurationTest.cs | 2 +-
.../Cache/Query/CacheQueriesTest.cs | 69 +++++++++++++++-----
.../Query/Linq/CacheLinqTest.Introspection.cs | 2 +-
.../Client/Cache/SqlQueryTest.cs | 4 +-
.../dotnet/Apache.Ignite.Core/Cache/ICache.cs | 8 +++
.../Impl/Binary/BinaryReaderExtensions.cs | 2 +-
.../Apache.Ignite.Core/Impl/Cache/CacheImpl.cs | 29 +++++---
.../Impl/Cache/ICacheInternal.cs | 2 +-
.../Impl/Cache/Query/FieldsQueryCursor.cs | 39 +++++++++++
.../Impl/Cache/Query/PlatformQueryQursorBase.cs | 8 +++
.../Impl/Client/Cache/CacheClient.cs | 2 +-
.../Cache/Query/ClientFieldsQueryCursor.cs | 12 +---
.../NuGet/LINQPad/BinaryModeExample.linq | 2 +-
.../NuGet/LINQPad/QueryExample.linq | 2 +-
.../Apache.Ignite.Linq/CacheLinqExtensions.cs | 10 +--
.../Impl/CacheFieldsQueryExecutor.cs | 10 +--
modules/platforms/dotnet/Apache.Ignite.ndproj | 2 +-
.../Datagrid/BinaryModeExample.cs | 5 +-
.../Datagrid/QueryDmlExample.cs | 18 ++---
.../Datagrid/QueryExample.cs | 5 +-
.../dotnet/examples/dotnetcore/App.config | 5 +-
.../dotnet/examples/dotnetcore/SqlExample.cs | 2 +-
32 files changed, 263 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
[48/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk-ce
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk-ce
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/64aa9eaa
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/64aa9eaa
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/64aa9eaa
Branch: refs/heads/ignite-zk
Commit: 64aa9eaabf5bce38a5703900365ec1a543f24c44
Parents: e419ec8 6e8cfe3
Author: sboikov <sb...@gridgain.com>
Authored: Wed Dec 20 10:52:32 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Dec 20 10:52:32 2017 +0300
----------------------------------------------------------------------
.../ignite/jdbc/JdbcErrorsAbstractSelfTest.java | 2 +-
.../ignite/jdbc/thin/JdbcThinBatchSelfTest.java | 5 +-
modules/dev-utils/README.txt | 5 +
.../ignite/development/utils/package-info.java | 2 +-
.../query/h2/DmlStatementsProcessor.java | 53 +++++-----
.../processors/query/h2/dml/DmlArgument.java | 31 ++++++
.../processors/query/h2/dml/DmlArguments.java | 104 +++++++++++++++++++
.../processors/query/h2/dml/DmlAstUtils.java | 14 ++-
.../processors/query/h2/dml/DmlUtils.java | 19 ++--
.../processors/query/h2/dml/FastUpdate.java | 91 ++--------------
.../query/h2/dml/FastUpdateArgument.java | 27 -----
.../processors/query/h2/dml/UpdatePlan.java | 80 +++++++++++---
.../query/h2/dml/UpdatePlanBuilder.java | 37 ++++++-
.../regression/KNNMultipleLinearRegression.java | 2 +-
14 files changed, 311 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
[19/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7e8f85ff
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7e8f85ff
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7e8f85ff
Branch: refs/heads/ignite-zk
Commit: 7e8f85ff81f1cfc0f163f86d623f7e558177aa2c
Parents: 376a484
Author: sboikov <sb...@gridgain.com>
Authored: Thu Dec 14 11:55:56 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Thu Dec 14 13:56:54 2017 +0300
----------------------------------------------------------------------
.../zk/internal/ZkAbstractChildrenCallback.java | 8 +
.../zk/internal/ZkCollectDistributedFuture.java | 171 ---------
.../ZkCommunicationErrorProcessFuture.java | 114 ++++--
...kCommunicationErrorResolveFinishMessage.java | 23 +-
.../ZkCommunicationErrorResolveResult.java | 37 ++
...ZkCommunicationErrorResolveStartMessage.java | 6 +
.../zk/internal/ZkDiscoveryCustomEventData.java | 19 +-
.../ZkDistributedCollectDataFuture.java | 223 ++++++++++++
.../discovery/zk/internal/ZkIgnitePaths.java | 20 +-
.../discovery/zk/internal/ZookeeperClient.java | 5 +
.../zk/internal/ZookeeperDiscoveryImpl.java | 360 +++++++++++--------
.../ZookeeperDiscoverySpiBasicTest.java | 176 ++++++++-
12 files changed, 808 insertions(+), 354 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
index 5679993..2292e35 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
@@ -48,6 +48,14 @@ abstract class ZkAbstractChildrenCallback extends ZkAbstractCallabck implements
}
}
+ /**
+ * @param rc
+ * @param path
+ * @param ctx
+ * @param children
+ * @param stat
+ * @throws Exception If failed.
+ */
abstract void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
throws Exception;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java
deleted file mode 100644
index fa529cf..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCollectDistributedFuture.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.zk.internal;
-
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.zookeeper.AsyncCallback;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-class ZkCollectDistributedFuture extends GridFutureAdapter<Void> {
- /** */
- private final IgniteLogger log;
-
- /** */
- private final String futPath;
-
- /** */
- private final ZookeeperDiscoveryImpl impl;
-
- /** */
- private final Set<Long> remainingNodes;
-
- /** */
- private final Callable<Void> lsnr;
-
- /**
- * @param impl
- * @param rtState
- * @param futPath
- */
- ZkCollectDistributedFuture(ZookeeperDiscoveryImpl impl, ZkRuntimeState rtState, String futPath, Callable<Void> lsnr) throws Exception {
- this.impl = impl;
- this.log = impl.log();
- this.futPath = futPath;
- this.lsnr = lsnr;
-
- ZkClusterNodes top = impl.nodes();
-
- remainingNodes = U.newHashSet(top.nodesByOrder.size());
-
- for (ZookeeperClusterNode node : top.nodesByInternalId.values())
- remainingNodes.add(node.order());
-
- NodeResultsWatcher watcher = new NodeResultsWatcher(rtState, impl);
-
- if (remainingNodes.isEmpty())
- completeAndNotifyListener();
- else
- rtState.zkClient.getChildrenAsync(futPath, watcher, watcher);
- }
-
- /**
- * @throws Exception If listener call failed.
- */
- private void completeAndNotifyListener() throws Exception {
- if (super.onDone())
- lsnr.call();
- }
-
- /**
- * @param futPath
- * @param client
- * @param nodeOrder
- * @param data
- * @throws Exception If failed.
- */
- static void saveNodeResult(String futPath, ZookeeperClient client, long nodeOrder, byte[] data) throws Exception {
- client.createIfNeeded(futPath + "/" + nodeOrder, data, CreateMode.PERSISTENT);
- }
-
- /**
- * @param node Failed node.
- */
- void onNodeFail(ZookeeperClusterNode node) throws Exception {
- long nodeOrder = node.order();
-
- if (remainingNodes.remove(nodeOrder)) {
- int remaining = remainingNodes.size();
-
- if (log.isInfoEnabled()) {
- log.info("ZkCollectDistributedFuture removed remaining failed node [node=" + nodeOrder +
- ", remaining=" + remaining +
- ", futPath=" + futPath + ']');
- }
-
- if (remaining == 0)
- completeAndNotifyListener();
- }
- }
-
- /**
- *
- */
- class NodeResultsWatcher extends ZkAbstractWatcher implements AsyncCallback.Children2Callback {
- /**
- * @param rtState Runtime state.
- * @param impl Discovery impl.
- */
- NodeResultsWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
- super(rtState, impl);
- }
-
- /** {@inheritDoc} */
- @Override protected void process0(WatchedEvent evt) {
- if (evt.getType() == Watcher.Event.EventType.NodeChildrenChanged)
- impl.zkClient().getChildrenAsync(evt.getPath(), this, this);
- }
-
- /** {@inheritDoc} */
- @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
- if (!onProcessStart())
- return;
-
- try {
- assert rc == 0 : KeeperException.Code.get(rc);
-
- if (isDone())
- return;
-
- for (int i = 0; i < children.size(); i++) {
- Long nodeOrder = Long.parseLong(children.get(i));
-
- if (remainingNodes.remove(nodeOrder)) {
- int remaining = remainingNodes.size();
-
- if (log.isInfoEnabled()) {
- log.info("ZkCollectDistributedFuture added new result [node=" + nodeOrder +
- ", remaining=" + remaining +
- ", futPath=" + path + ']');
- }
-
- if (remaining == 0)
- completeAndNotifyListener();
- }
- }
-
- onProcessEnd();
- }
- catch (Throwable e) {
- onProcessError(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index d87f500..15744a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -18,11 +18,10 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.util.Collection;
-import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
-import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -31,6 +30,7 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.jboss.netty.util.internal.ConcurrentHashMap;
import org.jetbrains.annotations.Nullable;
/**
@@ -41,7 +41,10 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
private final ZookeeperDiscoveryImpl impl;
/** */
- private final Map<Long, GridFutureAdapter<Boolean>> nodeFuts = new HashMap<>();
+ private final IgniteLogger log;
+
+ /** */
+ private final Map<Long, GridFutureAdapter<Boolean>> nodeFuts = new ConcurrentHashMap<>();
/** */
private final long endTime;
@@ -59,7 +62,10 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
private Set<Long> resFailedNodes;
/** */
- private ZkCollectDistributedFuture nodeResFut;
+ private Exception resErr;
+
+ /** */
+ private ZkDistributedCollectDataFuture collectResFut;
/**
* @param impl Discovery impl.
@@ -87,6 +93,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
assert state != State.DONE;
this.impl = impl;
+ this.log = impl.log();
if (state == State.WAIT_TIMEOUT) {
assert timeout > 0 : timeout;
@@ -102,15 +109,44 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
this.state = state;
}
- void nodeResultCollectFuture(ZkCollectDistributedFuture nodeResFut) {
- assert nodeResFut == null : nodeResFut;
+ /** {@inheritDoc} */
+ @Nullable @Override public IgniteLogger logger() {
+ return log;
+ }
- this.nodeResFut = nodeResFut;
+ /**
+ * @param collectResFut Collect nodes' communication status future.
+ */
+ void nodeResultCollectFuture(ZkDistributedCollectDataFuture collectResFut) {
+ assert this.collectResFut == null : collectResFut;
+
+ this.collectResFut = collectResFut;
}
+ /**
+ * @param top Topology.
+ * @throws Exception If failed.
+ */
+ void onTopologyChange(ZkClusterNodes top) throws Exception {
+ for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : nodeFuts.entrySet()) {
+ if (!top.nodesByOrder.containsKey(e.getKey()))
+ e.getValue().onDone(false);
+ }
+
+ if (collectResFut != null)
+ collectResFut.onTopologyChange(top);
+ }
+
+ /**
+ * @param locNodeOrder Local node order.
+ * @param rtState Runtime state.
+ * @param futPath Future path.
+ * @param nodes Nodes to ping.
+ * @throws Exception If failed.
+ */
void pingNodesAndNotifyFuture(long locNodeOrder, ZkRuntimeState rtState, String futPath, Collection<ClusterNode> nodes)
throws Exception {
- ZkCollectDistributedFuture.saveNodeResult(futPath, rtState.zkClient, locNodeOrder, null);
+ ZkDistributedCollectDataFuture.saveNodeResult(futPath, rtState.zkClient, locNodeOrder, null);
}
/**
@@ -146,6 +182,37 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
}
/**
+ * @param failedNodes Node failed as result of resolve process.
+ */
+ void onFinishResolve(Set<Long> failedNodes) {
+ Map<Long, GridFutureAdapter<Boolean>> futs;
+
+ synchronized (this) {
+ if (state == State.DONE) {
+ assert resErr != null;
+
+ return;
+ }
+
+ assert state == State.RESOLVE_STARTED : state;
+
+ state = State.DONE;
+
+ resFailedNodes = failedNodes;
+
+ futs = nodeFuts; // nodeFuts should not be modified after state changed to DONE.
+ }
+
+ for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : futs.entrySet()) {
+ Boolean res = !F.contains(resFailedNodes, e.getKey());
+
+ e.getValue().onDone(res);
+ }
+
+ onDone();
+ }
+
+ /**
* @param node Node.
* @return Future finished when communication error resolve is done or {@code null} if another
* resolve process should be started.
@@ -176,27 +243,19 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
return fut;
}
- /**
- * @param node Failed node.
- */
- void onNodeFailed(ClusterNode node) {
- GridFutureAdapter<Boolean> fut = null;
-
- synchronized (this) {
- if (state == State.WAIT_TIMEOUT)
- fut = nodeFuts.get(node.order());
- }
-
- if (fut != null)
- fut.onDone(false);
- }
-
/** {@inheritDoc} */
@Override public void run() {
// Run from zk discovery worker pool after timeout.
- if (processTimeout()) {
+ if (needProcessTimeout()) {
try {
- impl.sendCustomMessage(new ZkCommunicationErrorResolveStartMessage(UUID.randomUUID()));
+ UUID reqId = UUID.randomUUID();
+
+ if (log.isInfoEnabled()) {
+ log.info("Initiate cluster-wide communication error resolve process [reqId=" + reqId +
+ ", errNodes=" + nodeFuts.size() + ']');
+ }
+
+ impl.sendCustomMessage(new ZkCommunicationErrorResolveStartMessage(reqId));
}
catch (Exception e) {
Collection<GridFutureAdapter<Boolean>> futs;
@@ -206,6 +265,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
return;
state = State.DONE;
+ resErr = e;
futs = nodeFuts.values(); // nodeFuts should not be modified after state changed to DONE.
}
@@ -221,7 +281,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
/**
* @return {@code True} if need initiate resolve process after timeout expired.
*/
- private boolean processTimeout() {
+ private boolean needProcessTimeout() {
synchronized (this) {
if (state != State.WAIT_TIMEOUT)
return false;
@@ -251,7 +311,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
/** {@inheritDoc} */
@Override public void onTimeout() {
- if (processTimeout())
+ if (needProcessTimeout())
impl.runInWorkerThread(this);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
index 144a5bf..20aeddf 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
@@ -18,21 +18,42 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
/**
*
*/
-class ZkCommunicationErrorResolveFinishMessage implements ZkInternalMessage {
+class ZkCommunicationErrorResolveFinishMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
/** */
private static final long serialVersionUID = 0L;
/** */
final UUID futId;
+ /** */
+ transient ZkCommunicationErrorResolveResult res;
+
/**
* @param futId Future ID.
*/
ZkCommunicationErrorResolveFinishMessage(UUID futId) {
this.futId = futId;
}
+
+ /** {@inheritDoc} */
+ @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isMutable() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkCommunicationErrorResolveFinishMessage.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
new file mode 100644
index 0000000..745496b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.GridLongList;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkCommunicationErrorResolveResult implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ final GridLongList failedNodes;
+
+ ZkCommunicationErrorResolveResult(@Nullable GridLongList failedNodes) {
+ this.failedNodes = failedNodes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
index e619d7b..e85277b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
@@ -18,6 +18,7 @@
package org.apache.ignite.spi.discovery.zk.internal;
import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
import org.jetbrains.annotations.Nullable;
@@ -47,4 +48,9 @@ public class ZkCommunicationErrorResolveStartMessage implements DiscoverySpiCust
@Override public boolean isMutable() {
return false;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ZkCommunicationErrorResolveStartMessage.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
index 0d2288c..6375bc7 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
@@ -38,22 +38,33 @@ class ZkDiscoveryCustomEventData extends ZkDiscoveryEventData {
/** */
final String evtPath;
- /** */
- transient DiscoverySpiCustomMessage msg;
+ /** Message instance (can be marshalled as part of ZkDiscoveryCustomEventData or stored in separate znode. */
+ DiscoverySpiCustomMessage msg;
+
+ /** Unmarshalled message. */
+ transient DiscoverySpiCustomMessage resolvedMsg;
/**
* @param evtId Event ID.
* @param topVer Topology version.
* @param sndNodeId Sender node ID.
+ * @param msg Message instance.
* @param evtPath Event path.
* @param ack Acknowledge event flag.
*/
- ZkDiscoveryCustomEventData(long evtId, long topVer, UUID sndNodeId, String evtPath, boolean ack) {
+ ZkDiscoveryCustomEventData(long evtId,
+ long topVer,
+ UUID sndNodeId,
+ DiscoverySpiCustomMessage msg,
+ String evtPath,
+ boolean ack)
+ {
super(evtId, DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT, topVer);
assert sndNodeId != null;
- assert ack || !F.isEmpty(evtPath);
+ assert msg != null || ack || !F.isEmpty(evtPath);
+ this.msg = msg;
this.sndNodeId = sndNodeId;
this.evtPath = evtPath;
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
new file mode 100644
index 0000000..d33001b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
+ /** */
+ private final IgniteLogger log;
+
+ /** */
+ private final String futPath;
+
+ /** */
+ private final Set<Long> remainingNodes;
+
+ /** */
+ private final Callable<Void> lsnr;
+
+ /**
+ * @param impl Disovery impl
+ * @param rtState Runtime state.
+ * @param futPath Future path.
+ * @param lsnr Future listener.
+ * @throws Exception If listener call failed.
+ */
+ ZkDistributedCollectDataFuture(
+ ZookeeperDiscoveryImpl impl,
+ ZkRuntimeState rtState,
+ String futPath,
+ Callable<Void> lsnr)
+ throws Exception
+ {
+ this.log = impl.log();
+ this.futPath = futPath;
+ this.lsnr = lsnr;
+
+ ZkClusterNodes top = rtState.top;
+
+ remainingNodes = U.newHashSet(top.nodesByOrder.size());
+
+ for (ZookeeperClusterNode node : top.nodesByInternalId.values())
+ remainingNodes.add(node.order());
+
+ NodeResultsWatcher watcher = new NodeResultsWatcher(rtState, impl);
+
+ if (remainingNodes.isEmpty())
+ completeAndNotifyListener();
+ else {
+ if (log.isInfoEnabled())
+ log.info("Initialize data collect future [futPath=" + futPath + ", nodes=" + remainingNodes.size() + ']');
+
+ rtState.zkClient.getChildrenAsync(futPath, watcher, watcher);
+ }
+ }
+
+ /**
+ * @throws Exception If listener call failed.
+ */
+ private void completeAndNotifyListener() throws Exception {
+ if (super.onDone())
+ lsnr.call();
+ }
+
+ /**
+ * @param futPath
+ * @param client
+ * @param nodeOrder
+ * @param data
+ * @throws Exception If failed.
+ */
+ static void saveNodeResult(String futPath, ZookeeperClient client, long nodeOrder, byte[] data) throws Exception {
+ client.createIfNeeded(futPath + "/" + nodeOrder, data, CreateMode.PERSISTENT);
+ }
+
+ /**
+ * @param futResPath Result path.
+ * @param client Client.
+ * @param data Result data.
+ * @throws Exception If failed.
+ */
+ static void saveResult(String futResPath, ZookeeperClient client, byte[] data) throws Exception {
+ client.createIfNeeded(futResPath, data, CreateMode.PERSISTENT);
+ }
+
+ static byte[] readResult(ZookeeperClient client, ZkIgnitePaths paths, UUID futId) throws Exception {
+ return client.getData(paths.distributedFutureResultPath(futId));
+ }
+
+ /**
+ * @param client Client.
+ * @param paths Paths utils.
+ * @param futId Future ID.
+ * @throws Exception If failed.
+ */
+ static void deleteFutureData(ZookeeperClient client, ZkIgnitePaths paths, UUID futId) throws Exception {
+ // TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
+ String evtDir = paths.distributedFutureBasePath(futId);
+
+ client.deleteAll(evtDir,
+ client.getChildren(evtDir),
+ -1);
+
+ client.deleteIfExists(evtDir, -1);
+
+ client.deleteIfExists(paths.distributedFutureResultPath(futId), -1);
+ }
+
+ /**
+ * @param top Current topology.
+ * @throws Exception If listener call failed.
+ */
+ void onTopologyChange(ZkClusterNodes top) throws Exception {
+ if (remainingNodes.isEmpty())
+ return;
+
+ for (Iterator<Long> it = remainingNodes.iterator(); it.hasNext();) {
+ Long nodeOrder = it.next();
+
+ if (!top.nodesByOrder.containsKey(nodeOrder)) {
+ it.remove();
+
+ int remaining = remainingNodes.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("ZkDistributedCollectDataFuture removed remaining failed node [node=" + nodeOrder +
+ ", remaining=" + remaining +
+ ", futPath=" + futPath + ']');
+ }
+
+ if (remaining == 0) {
+ completeAndNotifyListener();
+
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ class NodeResultsWatcher extends ZkAbstractWatcher implements AsyncCallback.Children2Callback {
+ /**
+ * @param rtState Runtime state.
+ * @param impl Discovery impl.
+ */
+ NodeResultsWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+ super(rtState, impl);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void process0(WatchedEvent evt) {
+ if (evt.getType() == Watcher.Event.EventType.NodeChildrenChanged)
+ rtState.zkClient.getChildrenAsync(evt.getPath(), this, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ if (!onProcessStart())
+ return;
+
+ try {
+ if (!isDone()) {
+ assert rc == 0 : KeeperException.Code.get(rc);
+
+ for (int i = 0; i < children.size(); i++) {
+ Long nodeOrder = Long.parseLong(children.get(i));
+
+ if (remainingNodes.remove(nodeOrder)) {
+ int remaining = remainingNodes.size();
+
+ if (log.isInfoEnabled()) {
+ log.info("ZkDistributedCollectDataFuture added new result [node=" + nodeOrder +
+ ", remaining=" + remaining +
+ ", futPath=" + path + ']');
+ }
+
+ if (remaining == 0)
+ completeAndNotifyListener();
+ }
+ }
+ }
+
+ onProcessEnd();
+ }
+ catch (Throwable e) {
+ onProcessError(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
index 06c5d9e..2a1d804 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
@@ -26,22 +26,22 @@ class ZkIgnitePaths {
/** */
private static final int UUID_LEN = 36;
- /** */
+ /** Directory to store joined node data. */
private static final String JOIN_DATA_DIR = "jd";
- /** */
+ /** Directory to store new custom events. */
private static final String CUSTOM_EVTS_DIR = "ce";
- /** */
+ /** Directory to store parts of multi-parts custom events. */
private static final String CUSTOM_EVTS_PARTS_DIR = "cp";
- /** */
+ /** Directory to store acknowledge messages for custom events. */
private static final String CUSTOM_EVTS_ACKS_DIR = "ca";
- /** */
+ /** Directory to store EPHEMERAL znodes for alive cluster nodes. */
static final String ALIVE_NODES_DIR = "n";
- /** */
+ /** Path to store discovery events {@link ZkDiscoveryEventsData}. */
private static final String DISCO_EVENTS_PATH = "e";
/** */
@@ -300,4 +300,12 @@ class ZkIgnitePaths {
String distributedFutureBasePath(UUID id) {
return evtsPath + "/f-" + id;
}
+
+ /**
+ * @param id Future ID.
+ * @return Future path.
+ */
+ String distributedFutureResultPath(UUID id) {
+ return evtsPath + "/fr-" + id;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index a83886a..5923b39 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -321,6 +321,11 @@ public class ZookeeperClient implements Watcher {
return parts;
}
+ /**
+ * TODO ZK: it seems not always precise, e.g. if ACL is used?
+ * @param path Request path.
+ * @return Marshalled request overhead.
+ */
private int requestOverhead(String path) {
return path.length();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index a153d11..82d9c4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -54,7 +54,6 @@ import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.internal.LT;
import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.marshaller.MarshallerUtils;
@@ -72,7 +71,6 @@ import org.apache.zookeeper.AsyncCallback;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.data.Stat;
import org.jboss.netty.util.internal.ConcurrentHashMap;
import org.jetbrains.annotations.Nullable;
@@ -151,7 +149,10 @@ public class ZookeeperDiscoveryImpl {
public volatile IgniteDiscoverySpiInternalListener internalLsnr;
/** */
- private final ConcurrentHashMap<UUID, PingFuture> pingFuts = new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<Long, PingFuture> pingFuts = new ConcurrentHashMap<>();
+
+ /** */
+ private final AtomicReference<ZkCommunicationErrorProcessFuture> commErrProcFut = new AtomicReference<>();
/**
* @param spi Discovery SPI.
@@ -230,9 +231,9 @@ public class ZookeeperDiscoveryImpl {
return rtState.top.nodesByOrder.get(nodeOrder);
}
- /** */
- private final AtomicReference<ZkCommunicationErrorProcessFuture> commErrProcFut = new AtomicReference<>();
-
+ /**
+ * @param fut Future to remove.
+ */
void clearCommunicationErrorProcessFuture(ZkCommunicationErrorProcessFuture fut) {
assert fut.isDone() : fut;
@@ -262,6 +263,11 @@ public class ZookeeperDiscoveryImpl {
if (commErrProcFut.compareAndSet(fut, newFut)) {
fut = newFut;
+ if (log.isInfoEnabled()) {
+ log.info("Created new communication error process future [errNode=" + node0.id() +
+ ", err= " + err + ']');
+ }
+
fut.scheduleCheckOnTimeout();
}
else
@@ -295,7 +301,9 @@ public class ZookeeperDiscoveryImpl {
* @return Ping result.
*/
public boolean pingNode(UUID nodeId) {
- ZookeeperClusterNode node = node(nodeId);
+ ZkRuntimeState rtState = this.rtState;
+
+ ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
if (node == null)
return false;
@@ -303,12 +311,12 @@ public class ZookeeperDiscoveryImpl {
if (node.isLocal())
return true;
- PingFuture fut = pingFuts.get(nodeId);
+ PingFuture fut = pingFuts.get(node.order());
if (fut == null) {
- fut = new PingFuture(node);
+ fut = new PingFuture(rtState, node);
- PingFuture old = pingFuts.putIfAbsent(nodeId, fut);
+ PingFuture old = pingFuts.putIfAbsent(node.order(), fut);
if (old == null) {
if (fut.checkNodeAndState())
@@ -419,7 +427,7 @@ public class ZookeeperDiscoveryImpl {
connState = ConnectionState.STOPPED;
}
- zkClient().zk().sync(zkPaths.clusterDir, new SegmentedWatcher(), null);
+ rtState.zkClient.zk().sync(zkPaths.clusterDir, new SegmentedWatcher(), null);
}
else
joinFut.onDone(e);
@@ -530,33 +538,7 @@ public class ZookeeperDiscoveryImpl {
try {
ZookeeperClient zkClient = rtState.zkClient;
- String prefix = UUID.randomUUID().toString();
-
- int partCnt = 1;
-
- int overhead = 10;
-
- UUID locId = locNode.id();
-
- String path = zkPaths.createCustomEventPath(prefix, locId, partCnt);
-
- if (zkClient.needSplitNodeData(path, msgBytes, overhead)) {
- List<byte[]> parts = zkClient.splitNodeData(path, msgBytes, overhead);
-
- String partsBasePath = zkPaths.customEventPartsBasePath(prefix, locId);
-
- saveMultipleParts(zkClient, partsBasePath, parts);
-
- msgBytes = null;
-
- partCnt = parts.size();
- }
-
- zkClient.createSequential(prefix,
- zkPaths.customEvtsDir,
- zkPaths.createCustomEventPath(prefix, locId, partCnt),
- msgBytes,
- CreateMode.PERSISTENT_SEQUENTIAL);
+ saveCustomMessage(zkClient, msgBytes);
}
catch (ZookeeperClientFailedException e) {
if (clientReconnectEnabled)
@@ -575,6 +557,44 @@ public class ZookeeperDiscoveryImpl {
}
/**
+ * @param zkClient Client.
+ * @param msgBytes Marshalled message.
+ * @throws ZookeeperClientFailedException If connection to zk was lost.
+ * @throws InterruptedException If interrupted.
+ */
+ private void saveCustomMessage(ZookeeperClient zkClient, byte[] msgBytes)
+ throws ZookeeperClientFailedException, InterruptedException
+ {
+ String prefix = UUID.randomUUID().toString();
+
+ int partCnt = 1;
+
+ int overhead = 10;
+
+ UUID locId = locNode.id();
+
+ String path = zkPaths.createCustomEventPath(prefix, locId, partCnt);
+
+ if (zkClient.needSplitNodeData(path, msgBytes, overhead)) {
+ List<byte[]> parts = zkClient.splitNodeData(path, msgBytes, overhead);
+
+ String partsBasePath = zkPaths.customEventPartsBasePath(prefix, locId);
+
+ saveMultipleParts(zkClient, partsBasePath, parts);
+
+ msgBytes = null;
+
+ partCnt = parts.size();
+ }
+
+ zkClient.createSequential(prefix,
+ zkPaths.customEvtsDir,
+ zkPaths.createCustomEventPath(prefix, locId, partCnt),
+ msgBytes,
+ CreateMode.PERSISTENT_SEQUENTIAL);
+ }
+
+ /**
* @return Cluster start time.
*/
public long gridStartTime() {
@@ -859,7 +879,7 @@ public class ZookeeperDiscoveryImpl {
/**
*
*/
- private class CheckJoinStateTimeoutObject implements IgniteSpiTimeoutObject, Watcher, AsyncCallback.DataCallback {
+ private class CheckJoinStateTimeoutObject extends ZkAbstractWatcher implements IgniteSpiTimeoutObject, AsyncCallback.DataCallback {
/** */
private final IgniteUuid id = IgniteUuid.randomUuid();
@@ -869,16 +889,14 @@ public class ZookeeperDiscoveryImpl {
/** */
private final String joinDataPath;
- /** */
- private final ZkRuntimeState rtState;
-
/**
* @param joinDataPath Node joined data path.
* @param rtState State.
*/
CheckJoinStateTimeoutObject(String joinDataPath, ZkRuntimeState rtState) {
+ super(rtState, ZookeeperDiscoveryImpl.this);
+
this.joinDataPath = joinDataPath;
- this.rtState = rtState;
}
/** {@inheritDoc} */
@@ -909,7 +927,7 @@ public class ZookeeperDiscoveryImpl {
if (rc != 0)
return;
- if (!busyLock.enterBusy())
+ if (!onProcessStart())
return;
try {
@@ -921,27 +939,17 @@ public class ZookeeperDiscoveryImpl {
onSegmented(new IgniteSpiException(joinErr.err));
}
- busyLock.leaveBusy();
+ onProcessEnd();
}
catch (Throwable e) {
- onFatalError(busyLock, e);
+ onProcessError(e);
}
}
/** {@inheritDoc} */
- @Override public void process(WatchedEvent evt) {
- if (!busyLock.enterBusy())
- return;
-
- try {
- if (evt.getType() == Event.EventType.NodeDataChanged)
- rtState.zkClient.getDataAsync(evt.getPath(), this, this);
-
- busyLock.leaveBusy();
- }
- catch (Throwable e) {
- onFatalError(busyLock, e);
- }
+ @Override public void process0(WatchedEvent evt) {
+ if (evt.getType() == Event.EventType.NodeDataChanged)
+ rtState.zkClient.getDataAsync(evt.getPath(), this, this);
}
}
@@ -1243,18 +1251,20 @@ public class ZookeeperDiscoveryImpl {
return true;
}
else {
+ ZookeeperClient client = rtState.zkClient;
+
if (joinErr.notifyNode) {
String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
- zkClient().setData(joinDataPath, marshalZip(joinErr), -1);
+ client.setData(joinDataPath, marshalZip(joinErr), -1);
- zkClient().deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
+ client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
}
else {
if (log.isInfoEnabled())
log.info("Ignore join data, node was failed by previous coordinator: " + aliveNodePath);
- zkClient().deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
+ client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
}
return false;
@@ -1483,8 +1493,10 @@ public class ZookeeperDiscoveryImpl {
private void cleanupPreviousClusterData() throws Exception {
long start = System.currentTimeMillis();
- // TODO ZK: use multi, better batching.
- rtState.zkClient.setData(zkPaths.evtsPath, null, -1);
+ ZookeeperClient client = rtState.zkClient;
+
+ // TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
+ client.setData(zkPaths.evtsPath, null, -1);
List<String> evtChildren = rtState.zkClient.getChildren(zkPaths.evtsPath);
@@ -1494,10 +1506,10 @@ public class ZookeeperDiscoveryImpl {
removeChildren(evtDir);
}
- rtState.zkClient.deleteAll(zkPaths.evtsPath, evtChildren, -1);
+ client.deleteAll(zkPaths.evtsPath, evtChildren, -1);
- rtState.zkClient.deleteAll(zkPaths.customEvtsDir,
- rtState.zkClient.getChildren(zkPaths.customEvtsDir),
+ client.deleteAll(zkPaths.customEvtsDir,
+ client.getChildren(zkPaths.customEvtsDir),
-1);
rtState.zkClient.deleteAll(zkPaths.customEvtsPartsDir,
@@ -1525,20 +1537,6 @@ public class ZookeeperDiscoveryImpl {
}
/**
- * @return Nodes.
- */
- ZkClusterNodes nodes() {
- return rtState.top;
- }
-
- /**
- * @return Client.
- */
- ZookeeperClient zkClient() {
- return rtState.zkClient;
- }
-
- /**
* @param zkClient Client.
* @param evtPath Event path.
* @param sndNodeId Sender node ID.
@@ -1654,7 +1652,8 @@ public class ZookeeperDiscoveryImpl {
}
else {
if (log.isInfoEnabled()) {
- log.info("Start communication error resolve [sndNode=" + sndNode +
+ log.info("Start cluster-wide communication error resolve [sndNode=" + sndNode +
+ ", reqId=" + msg0.id +
", topVer=" + evtsData.topVer + ']');
}
@@ -1672,10 +1671,11 @@ public class ZookeeperDiscoveryImpl {
evtsData.evtIdGen,
evtsData.topVer,
sndNodeId,
+ null,
evtPath,
false);
- evtData.msg = msg;
+ evtData.resolvedMsg = msg;
evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
@@ -1746,7 +1746,7 @@ public class ZookeeperDiscoveryImpl {
(ZkDiscoveryCustomEventData)newEvts.evts.get(evtData.eventId());
if (evtData0 != null)
- evtData0.msg = ((ZkDiscoveryCustomEventData)evtData).msg;
+ evtData0.resolvedMsg = ((ZkDiscoveryCustomEventData)evtData).resolvedMsg;
}
}
}
@@ -1831,25 +1831,31 @@ public class ZookeeperDiscoveryImpl {
DiscoverySpiCustomMessage msg;
if (rtState.crd) {
- assert evtData0.msg != null : evtData0;
+ assert evtData0.resolvedMsg != null : evtData0;
- msg = evtData0.msg;
+ msg = evtData0.resolvedMsg;
}
else {
- if (evtData0.ackEvent()) {
- String path = zkPaths.ackEventDataPath(evtData0.eventId());
+ if (evtData0.msg == null) {
+ if (evtData0.ackEvent()) {
+ String path = zkPaths.ackEventDataPath(evtData0.eventId());
- msg = unmarshalZip(zkClient.getData(path));
- }
- else {
- byte[] msgBytes = readCustomEventData(zkClient,
- evtData0.evtPath,
- evtData0.sndNodeId);
+ msg = unmarshalZip(zkClient.getData(path));
+ }
+ else {
+ assert evtData0.evtPath != null : evtData0;
- msg = unmarshalZip(msgBytes);
+ byte[] msgBytes = readCustomEventData(zkClient,
+ evtData0.evtPath,
+ evtData0.sndNodeId);
+
+ msg = unmarshalZip(msgBytes);
+ }
}
+ else
+ msg = evtData0.msg;
- evtData0.msg = msg;
+ evtData0.resolvedMsg = msg;
}
if (msg instanceof ZkInternalMessage)
@@ -1889,6 +1895,11 @@ public class ZookeeperDiscoveryImpl {
zkClient.setData(rtState.locNodeZkPath, marshalZip(rtState.locNodeInfo), -1);
}
+
+ ZkCommunicationErrorProcessFuture commErrFut = commErrProcFut.get();
+
+ if (commErrFut != null)
+ commErrFut.onTopologyChange(rtState.top); // This can add new event, notify out of event process loop.
}
/**
@@ -1973,12 +1984,13 @@ public class ZookeeperDiscoveryImpl {
if (msg instanceof ZkForceNodeFailMessage)
processForceNodeFailMessage((ZkForceNodeFailMessage)msg, evtData);
else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
- processStartResolveCommunicationErrorMessage(
+ processCommunicationErrorResolveStartMessage(
(ZkCommunicationErrorResolveStartMessage)msg,
evtData);
}
else if (msg instanceof ZkCommunicationErrorResolveFinishMessage) {
- ZkCommunicationErrorResolveFinishMessage msg0 = (ZkCommunicationErrorResolveFinishMessage)msg;
+ processCommunicationErrorResolveFinishMessage(
+ (ZkCommunicationErrorResolveFinishMessage)msg);
}
}
@@ -2012,9 +2024,48 @@ public class ZookeeperDiscoveryImpl {
/**
* @param msg Message.
+ * @throws Exception If failed.
+ */
+ private void processCommunicationErrorResolveFinishMessage(ZkCommunicationErrorResolveFinishMessage msg)
+ throws Exception
+ {
+ UUID futId = msg.futId;
+
+ assert futId != null;
+ assert futId.equals(rtState.evtsData.communicationErrorResolveFutureId());
+
+ if (log.isInfoEnabled())
+ log.info("Received communication error resolve finish message [reqId=" + futId + ']');
+
+ rtState.evtsData.communicationErrorResolveFutureId(null);
+
+ ZkCommunicationErrorResolveResult res = msg.res;
+
+ if (res == null)
+ res = unmarshalZip(ZkDistributedCollectDataFuture.readResult(rtState.zkClient, zkPaths, futId));
+
+ ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+ assert fut != null;
+
+ Set<Long> failedNodes = null;
+
+ if (res.failedNodes != null) {
+ failedNodes = U.newHashSet(res.failedNodes.size());
+
+ for (int i = 0; i < res.failedNodes.size(); i++)
+ failedNodes.add(res.failedNodes.get(i));
+ }
+
+ fut.onFinishResolve(failedNodes);
+ }
+
+ /**
+ * @param msg Message.
* @param evtData Event data.
+ * @throws Exception If failed.
*/
- private void processStartResolveCommunicationErrorMessage(ZkCommunicationErrorResolveStartMessage msg,
+ private void processCommunicationErrorResolveStartMessage(ZkCommunicationErrorResolveStartMessage msg,
ZkDiscoveryCustomEventData evtData) throws Exception {
ZkCommunicationErrorProcessFuture fut;
@@ -2043,6 +2094,11 @@ public class ZookeeperDiscoveryImpl {
}
}
+ if (log.isInfoEnabled()) {
+ log.info("Received communication error resolve request [reqId=" + msg.id +
+ ", topVer=" + rtState.top.topologySnapshot() + ']');
+ }
+
assert !fut.isDone() : fut;
final String futPath = zkPaths.distributedFutureBasePath(msg.id);
@@ -2050,7 +2106,7 @@ public class ZookeeperDiscoveryImpl {
final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
if (rtState.crd) {
- ZkCollectDistributedFuture nodeResFut = new ZkCollectDistributedFuture(this, rtState, futPath,
+ ZkDistributedCollectDataFuture nodeResFut = new ZkDistributedCollectDataFuture(this, rtState, futPath,
new Callable<Void>() {
@Override public Void call() throws Exception {
// Future is completed from ZK event thread.
@@ -2075,25 +2131,34 @@ public class ZookeeperDiscoveryImpl {
* @param rtState Runtime state.
* @throws Exception If failed.
*/
- void finishCommunicationResolveProcess(ZkRuntimeState rtState) throws Exception {
+ private void finishCommunicationResolveProcess(ZkRuntimeState rtState) throws Exception {
ZkDiscoveryEventsData evtsData = rtState.evtsData;
UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
assert futId != null;
- rtState.evtsData.communicationErrorResolveFutureId(null);
-
ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId);
+ ZkCommunicationErrorResolveResult res = new ZkCommunicationErrorResolveResult(null);
+
+ msg.res = res;
+
+ ZkDistributedCollectDataFuture.saveResult(zkPaths.distributedFutureResultPath(futId),
+ rtState.zkClient,
+ marshalZip(res));
+
+ evtsData.evtIdGen++;
+
ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
evtsData.evtIdGen,
evtsData.topVer,
locNode.id(),
+ msg,
null,
false);
- evtData.msg = msg;
+ evtData.resolvedMsg = msg;
evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
@@ -2104,11 +2169,15 @@ public class ZookeeperDiscoveryImpl {
*
*/
public void simulateNodeFailure() {
- zkClient().deleteIfExistsAsync(zkPaths.aliveNodesDir);
+ ZkRuntimeState rtState = this.rtState;
+
+ ZookeeperClient client = rtState.zkClient;
+
+ client.deleteIfExistsAsync(zkPaths.aliveNodesDir);
rtState.onCloseStart();
- zkClient().close();
+ rtState.zkClient.close();
}
/**
@@ -2211,16 +2280,11 @@ public class ZookeeperDiscoveryImpl {
throw new ZookeeperClientFailedException("Received node failed event for local node.");
}
else {
- PingFuture pingFut = pingFuts.get(failedNode.id());
+ PingFuture pingFut = pingFuts.get(failedNode.order());
if (pingFut != null)
pingFut.onDone(false);
- ZkCommunicationErrorProcessFuture commErrFut = commErrProcFut.get();
-
- if (commErrFut != null)
- commErrFut.onNodeFailed(failedNode);
-
final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
lsnr.onDiscovery(EVT_NODE_FAILED,
@@ -2291,9 +2355,10 @@ public class ZookeeperDiscoveryImpl {
evtData.topologyVersion(), // Use topology version from original event.
locNode.id(),
null,
+ null,
true);
- ackEvtData.msg = ack;
+ ackEvtData.resolvedMsg = ack;
if (newEvts == null)
newEvts = new ArrayList<>();
@@ -2404,18 +2469,28 @@ public class ZookeeperDiscoveryImpl {
* @param ctx Context for log.
* @param evtData Event data.
* @return Ack message.
+ * @throws Exception If failed.
*/
- @Nullable private DiscoverySpiCustomMessage handleProcessedCustomEvent(String ctx, ZkDiscoveryCustomEventData evtData) {
+ @Nullable private DiscoverySpiCustomMessage handleProcessedCustomEvent(String ctx, ZkDiscoveryCustomEventData evtData)
+ throws Exception {
if (log.isDebugEnabled())
log.debug("All nodes processed custom event [ctx=" + ctx + ", evtData=" + evtData + ']');
if (!evtData.ackEvent()) {
- deleteCustomEventDataAsync(rtState.zkClient, evtData.evtPath);
+ if (evtData.evtPath != null)
+ deleteCustomEventDataAsync(rtState.zkClient, evtData.evtPath);
+ else {
+ if (evtData.resolvedMsg instanceof ZkCommunicationErrorResolveFinishMessage) {
+ UUID futId = ((ZkCommunicationErrorResolveFinishMessage)evtData.resolvedMsg).futId;
- assert evtData.msg != null || locNode.order() > evtData.topologyVersion() : evtData;
+ ZkDistributedCollectDataFuture.deleteFutureData(rtState.zkClient, zkPaths, futId);
+ }
+ }
- if (evtData.msg != null)
- return evtData.msg.ackMessage();
+ assert evtData.resolvedMsg != null || locNode.order() > evtData.topologyVersion() : evtData;
+
+ if (evtData.resolvedMsg != null)
+ return evtData.resolvedMsg.ackMessage();
}
else {
String path = zkPaths.ackEventDataPath(evtData.eventId());
@@ -2829,7 +2904,7 @@ public class ZookeeperDiscoveryImpl {
/**
*
*/
- private class PingFuture extends GridFutureAdapter<Boolean> implements IgniteSpiTimeoutObject, Runnable {
+ private class PingFuture extends GridFutureAdapter<Boolean> implements IgniteSpiTimeoutObject {
/** */
private final ZookeeperClusterNode node;
@@ -2839,16 +2914,20 @@ public class ZookeeperDiscoveryImpl {
/** */
private final IgniteUuid id;
+ /** */
+ private final ZkRuntimeState rtState;
+
/**
* @param node Node.
*/
- PingFuture(ZookeeperClusterNode node) {
+ PingFuture(ZkRuntimeState rtState, ZookeeperClusterNode node) {
+ this.rtState = rtState;
this.node = node;
id = IgniteUuid.fromUuid(node.id());
endTime = System.currentTimeMillis() + node.sessionTimeout() + 1000;
- }
+ };
/** {@inheritDoc} */
@Override public IgniteUuid id() {
@@ -2861,33 +2940,34 @@ public class ZookeeperDiscoveryImpl {
}
/** {@inheritDoc} */
- @Override public void run() {
+ @Override public void onTimeout() {
if (checkNodeAndState()) {
- try {
- for (String path : zkClient().getChildren(zkPaths.aliveNodesDir)) {
- if (node.internalId() == ZkIgnitePaths.aliveInternalId(path)) {
- onDone(true);
+ runInWorkerThread(new ZkRunnable(rtState, ZookeeperDiscoveryImpl.this) {
+ @Override protected void run0() throws Exception {
+ if (checkNodeAndState()) {
+ try {
+ for (String path : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+ if (node.internalId() == ZkIgnitePaths.aliveInternalId(path)) {
+ onDone(true);
+
+ return;
+ }
+ }
+
+ onDone(false);
+ }
+ catch (Exception e) {
+ onDone(e);
- return;
+ throw e;
+ }
}
}
-
- onDone(false);
- }
- catch (Exception e) {
- if (checkNodeAndState())
- onDone(e);
- }
+ });
}
}
/** {@inheritDoc} */
- @Override public void onTimeout() {
- if (checkNodeAndState())
- runInWorkerThread(this);
- }
-
- /** {@inheritDoc} */
@Override public boolean onDone(@Nullable Boolean res, @Nullable Throwable err) {
if (super.onDone(res, err)) {
pingFuts.remove(node.id(), this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7e8f85ff/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index 3f6a8dc..ec70be6 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -34,6 +34,7 @@ import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.curator.test.InstanceSpec;
import org.apache.curator.test.TestingCluster;
import org.apache.curator.test.TestingZooKeeperServer;
@@ -70,6 +71,8 @@ import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.logger.java.JavaLogger;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.CommunicationProblemContext;
+import org.apache.ignite.spi.discovery.CommunicationProblemResolver;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
import org.apache.ignite.testframework.GridTestUtils;
@@ -135,6 +138,9 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/** */
private boolean persistence;
+ /** */
+ private CommunicationProblemResolver communicationProblemResolver;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
if (testSockNio)
@@ -170,8 +176,6 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
cfg.setCacheConfiguration(ccfg);
- // cfg.setMarshaller(new JdkMarshaller());
-
cfg.setClientMode(client);
if (userAttrs != null)
@@ -248,6 +252,10 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
}
+ /**
+ * @param instances Number of instances.
+ * @return Cluster.
+ */
private static TestingCluster createTestingCluster(int instances) {
String tmpDir = System.getProperty("java.io.tmpdir");
@@ -270,6 +278,9 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
return new TestingCluster(specs);
}
+ /**
+ * @param file Directory to delete.
+ */
private static void deleteRecursively0(File file) {
File[] files = file.listFiles();
@@ -279,8 +290,10 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
for (File f : files) {
if (f.isDirectory())
deleteRecursively0(f);
- else
- f.delete();
+ else {
+ if (!f.delete())
+ throw new IgniteException("Failed to delete file: " + f.getAbsolutePath());
+ }
}
}
@@ -331,6 +344,8 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
assertFalse("Unexpected error, see log for details", err);
checkEventsConsistency();
+
+ checkInternalStructuresCleanup();
}
finally {
reset();
@@ -342,6 +357,23 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ private void checkInternalStructuresCleanup() throws Exception {
+ for (Ignite node : G.allGrids()) {
+ final AtomicReference<?> res = GridTestUtils.getFieldValue(spi(node), "impl", "commErrProcFut");
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return res.get() == null;
+ }
+ }, 5000);
+
+ assertNull(res.get());
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testMetadataUpdate() throws Exception {
startGrid(0);
@@ -879,16 +911,31 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
waitForTopology(initNodes + startNodes - failCnt);
}
+ /**
+ * @param node
+ * @return
+ */
private static String aliveZkNodePath(Ignite node) {
return aliveZkNodePath(node.configuration().getDiscoverySpi());
}
+ /**
+ * @param spi
+ * @return
+ */
private static String aliveZkNodePath(DiscoverySpi spi) {
String path = GridTestUtils.getFieldValue(spi, "impl", "rtState", "locNodeZkPath");
return path.substring(path.lastIndexOf('/') + 1);
}
+ /**
+ * @param log
+ * @param connectString
+ * @param failedZkNodes
+ * @param timeout
+ * @throws Exception
+ */
private static void waitNoAliveZkNodes(final IgniteLogger log,
String connectString, final List<String> failedZkNodes,
long timeout)
@@ -1724,6 +1771,99 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_1() throws Exception {
+ communicationErrorResolve_Simple(2);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_2() throws Exception {
+ communicationErrorResolve_Simple(10);
+ }
+
+ /**
+ * @param nodes Nodes number.
+ * @throws Exception If failed.
+ */
+ private void communicationErrorResolve_Simple(int nodes) throws Exception {
+ assert nodes > 1;
+
+ sesTimeout = 2000;
+ communicationProblemResolver = new NoOpCommunicationProblemResolver();
+
+ startGridsMultiThreaded(nodes);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 3; i++) {
+ info("Iteration: " + i);
+
+ int idx1 = rnd.nextInt(nodes);
+
+ int idx2;
+
+ do {
+ idx2 = rnd.nextInt(nodes);
+ }
+ while (idx1 == idx2);
+
+ ZookeeperDiscoverySpi spi = spi(ignite(idx1));
+
+ spi.onCommunicationConnectionError(ignite(idx2).cluster().localNode(), new Exception("test"));
+
+ checkInternalStructuresCleanup();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNoOpCommunicationErrorResolve_3() throws Exception {
+ // One node fails before sending communication status.
+ sesTimeout = 2000;
+ communicationProblemResolver = new NoOpCommunicationProblemResolver();
+
+ startGridsMultiThreaded(3);
+
+ sesTimeout = 10_000;
+
+ testSockNio = true;
+ sesTimeout = 5000;
+
+ startGrid(3);
+
+ IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() {
+ ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+ spi.onCommunicationConnectionError(ignite(1).cluster().localNode(), new Exception("test"));
+
+ return null;
+ }
+ });
+
+ U.sleep(1000);
+
+ ZkTestClientCnxnSocketNIO nio = ZkTestClientCnxnSocketNIO.forNode(ignite(3));
+
+ nio.closeSocket(true);
+
+ try {
+ stopGrid(3);
+
+ fut.get();
+ }
+ finally {
+ nio.allowConnect();
+ }
+
+ waitForTopology(3);
+ }
+
+ /**
* @param dfltConsistenId Default consistent ID flag.
* @throws Exception If failed.
*/
@@ -2062,6 +2202,14 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * @param node Node.
+ * @return Node's discovery SPI.
+ */
+ private static ZookeeperDiscoverySpi spi(Ignite node) {
+ return (ZookeeperDiscoverySpi)node.configuration().getDiscoverySpi();
+ }
+
+ /**
* @param nodeName Node name.
* @return Node's discovery SPI.
* @throws Exception If failed.
@@ -2069,7 +2217,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
private ZookeeperDiscoverySpi waitSpi(final String nodeName) throws Exception {
GridTestUtils.waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
- return spis.contains(nodeName);
+ return spis.containsKey(nodeName);
}
}, 5000);
@@ -2080,6 +2228,10 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
return spi;
}
+ /**
+ * @param topVer Topology version.
+ * @return Expected event instance.
+ */
private static DiscoveryEvent joinEvent(long topVer) {
DiscoveryEvent expEvt = new DiscoveryEvent(null, null, EventType.EVT_NODE_JOINED, null);
@@ -2088,6 +2240,10 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
return expEvt;
}
+ /**
+ * @param topVer Topology version.
+ * @return Expected event instance.
+ */
private static DiscoveryEvent failEvent(long topVer) {
DiscoveryEvent expEvt = new DiscoveryEvent(null, null, EventType.EVT_NODE_FAILED, null);
@@ -2370,4 +2526,14 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
private static class C2 implements Serializable {
// No-op.
}
+
+ /**
+ *
+ */
+ static class NoOpCommunicationProblemResolver implements CommunicationProblemResolver {
+ /** {@inheritDoc} */
+ @Override public void resolve(CommunicationProblemContext ctx) {
+ // No-op.
+ }
+ }
}
[29/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/60b625a9
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/60b625a9
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/60b625a9
Branch: refs/heads/ignite-zk
Commit: 60b625a99f956db75f1c2abfdbeb59b1df91166d
Parents: 0b22522
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 14:16:36 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 14:16:36 2017 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 6 ++
.../ignite/internal/util/nio/GridNioServer.java | 2 +-
.../communication/tcp/TcpCommunicationSpi.java | 69 ++++++-------
.../TcpCommunicationConnectionCheckFuture.java | 100 +++++++++++++++++++
.../ZkCommunicationErrorProcessFuture.java | 2 +-
.../GridTcpCommunicationSpiAbstractTest.java | 38 ++++++-
.../ZookeeperDiscoverySpiBasicTest.java | 6 +-
7 files changed, 177 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 7761020..37a7d8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -155,6 +155,7 @@ import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageFactory;
import org.apache.ignite.spi.collision.jobstealing.JobStealingRequest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationConnectionCheckMessage;
import org.apache.ignite.spi.communication.tcp.HandshakeMessage;
import org.apache.ignite.spi.communication.tcp.HandshakeMessage2;
import org.apache.ignite.spi.communication.tcp.NodeIdMessage;
@@ -186,6 +187,11 @@ public class GridIoMessageFactory implements MessageFactory {
switch (type) {
// -54 is reserved for SQL.
// -46 ... -51 - snapshot messages.
+ case -62:
+ msg = new TcpCommunicationConnectionCheckMessage();
+
+ break;
+
case -61:
msg = new IgniteDiagnosticMessage();
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 14d55d8..9784549 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -842,7 +842,7 @@ public class GridNioServer<T> {
NioOperationFuture<GridNioSession> req = new NioOperationFuture<>(ch, false, meta);
if (async) {
- // assert meta != null;
+ assert meta != null;
req.op = NioOperation.CONNECT;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index ca73e7b..d4ccc33 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -135,6 +135,7 @@ import org.apache.ignite.spi.IgniteSpiThread;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.CommunicationListener;
import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.thread.IgniteThread;
@@ -308,11 +309,17 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT = IgniteProductVersion.fromString("2.1.4");
/** Connection index meta for session. */
- private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
+ public static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
/** Message tracker meta for session. */
private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
+ /** Session future. */
+ public static final int SES_FUT_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** */
+ public static final ConnectionKey CONN_CHECK_DUMMY_KEY = new ConnectionKey(null, -1, -1);
+
/**
* Default local port range (value is <tt>100</tt>).
* See {@link #setLocalPortRange(int)} for details.
@@ -396,9 +403,13 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
else {
- if (log.isInfoEnabled())
- log.info("Established outgoing communication connection [locAddr=" + ses.localAddress() +
- ", rmtAddr=" + ses.remoteAddress() + ']');
+ ConnectionKey connId = ses.meta(CONN_IDX_META);
+
+ if (connId != CONN_CHECK_DUMMY_KEY) {
+ if (log.isInfoEnabled())
+ log.info("Established outgoing communication connection [locAddr=" + ses.localAddress() +
+ ", rmtAddr=" + ses.remoteAddress() + ']');
+ }
}
}
@@ -676,7 +687,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ConnectionKey connKey = ses.meta(CONN_IDX_META);
if (connKey == null) {
- assert ses.accepted() : ses;
+ assert ses.accepted() : msg;
if (!connectGate.tryEnter()) {
if (log.isDebugEnabled())
@@ -736,6 +747,17 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
recovery.lastAcknowledged(rcvCnt);
}
}
+ else if (connKey == CONN_CHECK_DUMMY_KEY) {
+ assert msg instanceof NodeIdMessage : msg;
+
+ TcpCommunicationConnectionCheckFuture fut = ses.meta(SES_FUT_META);
+
+ fut.onConnected(U.bytesToUuid(((NodeIdMessage) msg).nodeIdBytes, 0));
+
+ nioSrvr.closeFromWorkerThread(ses);
+
+ return;
+ }
}
IgniteRunnable c;
@@ -2566,45 +2588,12 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
sendMessage0(node, msg, null);
}
- public IgniteFuture<BitSet> pingNodes(List<ClusterNode> nodes) {
+ public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
ClusterNode node = nodes.get(0);
try {
- LinkedHashSet<InetSocketAddress> addrs = nodeAddresses(node);
-
- // /172.25.4.90:45012
+ Collection<InetSocketAddress> addrs = nodeAddresses(node);
- for (InetSocketAddress addr : addrs) {
- SocketChannel ch = SocketChannel.open();
-
- ch.configureBlocking(false);
-
- ch.socket().setTcpNoDelay(tcpNoDelay);
- ch.socket().setKeepAlive(true);
-
- boolean connect = ch.connect(addr);
-
- if (!connect) {
- GridNioFuture<GridNioSession> fut = nioSrvr.createSession(ch, null, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
- @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
- try {
- GridNioSession ses = fut.get();
-
- log.info("Ping connected");
-
- nioSrvr.closeFromWorkerThread(ses);
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- fut.get();
- }
- else
- log.info("Connected");
- }
}
catch (Exception e) {
throw new IgniteSpiException(e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
new file mode 100644
index 0000000..e08ea13
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.SocketChannel;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.GridLeanMap;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.nio.GridNioServer;
+import org.apache.ignite.internal.util.nio.GridNioSession;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+/**
+ *
+ */
+public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Boolean> {
+ /** */
+ private final UUID nodeId;
+
+ /** */
+ private final GridNioServer nioSrvr;
+
+ /** */
+ private Map<Integer, Object> sesMeta;
+
+ /** */
+ private SocketChannel ch;
+
+ /**
+ * @param nodeId Remote note ID.
+ */
+ public TcpCommunicationConnectionCheckFuture(GridNioServer nioSrvr, UUID nodeId) {
+ this.nioSrvr = nioSrvr;
+ this.nodeId = nodeId;
+ }
+
+ /**
+ * @param addr
+ * @throws IOException
+ */
+ public void init(InetSocketAddress addr) throws IOException {
+ ch = SocketChannel.open();
+
+ ch.configureBlocking(false);
+
+ ch.socket().setTcpNoDelay(true);
+ ch.socket().setKeepAlive(false);
+
+ boolean connect = ch.connect(addr);
+
+ if (!connect) {
+ sesMeta = new GridLeanMap<>(2);
+
+ sesMeta.put(TcpCommunicationSpi.CONN_IDX_META, TcpCommunicationSpi.CONN_CHECK_DUMMY_KEY);
+ sesMeta.put(TcpCommunicationSpi.SES_FUT_META, this);
+
+ nioSrvr.createSession(ch, sesMeta, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
+ @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
+ if (fut.error() != null)
+ onDone(false);
+ }
+ });
+ }
+ }
+
+ /**
+ *
+ */
+ public void onTimeout() {
+ if (super.onDone(false))
+ nioSrvr.cancelConnect(ch, sesMeta);
+ }
+
+ /**
+ * @param rmtNodeId
+ */
+ public void onConnected(UUID rmtNodeId) {
+ onDone(nodeId.equals(rmtNodeId));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index 6812ab0..d7d4bd1 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -151,7 +151,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
throws Exception {
TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
- spi.pingNodes(nodes);
+ spi.checkConnection(nodes);
ZkDistributedCollectDataFuture.saveNodeResult(futPath, rtState.zkClient, locNodeOrder, null);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
index 54b3a78..caa2d87 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
@@ -17,14 +17,21 @@
package org.apache.ignite.spi.communication.tcp;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.IgniteSpiAdapter;
import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.GridAbstractCommunicationSelfTest;
+import org.apache.ignite.spi.communication.GridTestMessage;
import org.apache.ignite.testframework.GridTestUtils;
/**
@@ -32,7 +39,7 @@ import org.apache.ignite.testframework.GridTestUtils;
*/
abstract class GridTcpCommunicationSpiAbstractTest extends GridAbstractCommunicationSelfTest<CommunicationSpi> {
/** */
- private static final int SPI_COUNT = 3;
+ private static final int SPI_COUNT = 2;
/** */
public static final int IDLE_CONN_TIMEOUT = 2000;
@@ -85,6 +92,35 @@ abstract class GridTcpCommunicationSpiAbstractTest extends GridAbstractCommunica
}
}
+ public void testConnectionCheck() {
+ for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+ UUID id = entry.getKey();
+
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+
+ List<ClusterNode> checkNodes = new ArrayList<>();
+
+ for (ClusterNode node : nodes) {
+ if (!id.equals(node.id()))
+ checkNodes.add(node);
+ }
+
+ spi.checkConnection(checkNodes);
+
+ break;
+// for (ClusterNode node : nodes) {
+// synchronized (mux) {
+// if (!msgDestMap.containsKey(entry.getKey()))
+// msgDestMap.put(entry.getKey(), new HashSet<UUID>());
+//
+// msgDestMap.get(entry.getKey()).add(node.id());
+// }
+//
+// entry.getValue().sendMessage(node, new GridTestMessage(entry.getKey(), msgId++, 0));
+// }
+ }
+ }
+
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
super.afterTest();
http://git-wip-us.apache.org/repos/asf/ignite/blob/60b625a9/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index 64fcd34..a2e8784 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -1929,7 +1929,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
nodes.add(ignite(2).cluster().localNode());
- spi.pingNodes(nodes);
+ // spi.pingNodes(nodes);
}
/**
@@ -2628,7 +2628,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
- @Override public IgniteFuture<BitSet> pingNodes(List<ClusterNode> nodes) {
+ @Override public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
CountDownLatch pingLatch = this.pingLatch;
try {
@@ -2639,7 +2639,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
throw new IgniteException(e);
}
- return super.pingNodes(nodes);
+ return super.checkConnection(nodes);
}
}
}
[32/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ed492a4f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ed492a4f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ed492a4f
Branch: refs/heads/ignite-zk
Commit: ed492a4f3aca046b6e196420bd986d0f47232aed
Parents: 78a994a
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 16:00:49 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 17:08:26 2017 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 17 -
.../communication/tcp/TcpCommunicationSpi.java | 148 ++-----
.../tcp/internal/ConnectionKey.java | 92 ++++
.../TcpCommunicationConnectionCheckFuture.java | 423 +++++++++++++++++--
...pCommunicationNodeConnectionCheckFuture.java | 30 ++
.../GridTcpCommunicationSpiAbstractTest.java | 83 ++--
6 files changed, 618 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed492a4f/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index a30c439..78cb7a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -117,8 +117,6 @@ import org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse;
import org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx;
-import org.apache.ignite.internal.processors.cluster.ClusterMetricsUpdateMessage;
-import org.apache.ignite.internal.processors.continuous.ContinuousRoutineStartResultMessage;
import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
@@ -186,11 +184,6 @@ public class GridIoMessageFactory implements MessageFactory {
switch (type) {
// -54 is reserved for SQL.
// -46 ... -51 - snapshot messages.
- case -62:
- msg = new TcpCommunicationConnectionCheckMessage();
-
- break;
-
case -61:
msg = new IgniteDiagnosticMessage();
@@ -886,16 +879,6 @@ public class GridIoMessageFactory implements MessageFactory {
break;
- case 129:
- msg = new ClusterMetricsUpdateMessage();
-
- break;
-
- case 130:
- msg = new ContinuousRoutineStartResultMessage();
-
- break;
-
// [-3..119] [124..128] [-23..-27] [-36..-55]- this
// [120..123] - DR
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed492a4f/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 3588a79..afa6953 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -72,6 +72,7 @@ import org.apache.ignite.internal.util.GridConcurrentFactory;
import org.apache.ignite.internal.util.GridSpinReadWriteLock;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
import org.apache.ignite.internal.util.ipc.IpcEndpoint;
import org.apache.ignite.internal.util.ipc.IpcToNioAdapter;
import org.apache.ignite.internal.util.ipc.shmem.IpcOutOfSystemResourcesException;
@@ -82,7 +83,6 @@ import org.apache.ignite.internal.util.nio.GridConnectionBytesVerifyFilter;
import org.apache.ignite.internal.util.nio.GridDirectParser;
import org.apache.ignite.internal.util.nio.GridNioCodecFilter;
import org.apache.ignite.internal.util.nio.GridNioFilter;
-import org.apache.ignite.internal.util.nio.GridNioFuture;
import org.apache.ignite.internal.util.nio.GridNioMessageReaderFactory;
import org.apache.ignite.internal.util.nio.GridNioMessageTracker;
import org.apache.ignite.internal.util.nio.GridNioMessageWriterFactory;
@@ -135,11 +135,13 @@ import org.apache.ignite.spi.IgniteSpiThread;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.CommunicationListener;
import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.internal.ConnectionKey;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationNodeConnectionCheckFuture;
import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage;
import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
import org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage;
import org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage;
-import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.thread.IgniteThread;
@@ -149,6 +151,8 @@ import org.jsr166.ConcurrentLinkedDeque8;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
+import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.CONN_CHECK_DUMMY_KEY;
+import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.SES_FUT_META;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NEED_WAIT;
import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NODE_STOPPING;
@@ -318,12 +322,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/** Message tracker meta for session. */
private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
- /** Session future. */
- public static final int SES_FUT_META = GridNioSessionMetaKey.nextUniqueKey();
-
- /** */
- public static final ConnectionKey CONN_CHECK_DUMMY_KEY = new ConnectionKey(null, -1, -1);
-
/**
* Default local port range (value is <tt>100</tt>).
* See {@link #setLocalPortRange(int)} for details.
@@ -409,17 +407,18 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
else {
ConnectionKey connId = ses.meta(CONN_IDX_META);
- if (connId != CONN_CHECK_DUMMY_KEY) {
- if (log.isInfoEnabled())
- log.info("Established outgoing communication connection [locAddr=" + ses.localAddress() +
- ", rmtAddr=" + ses.remoteAddress() + ']');
- }
+ if (log.isInfoEnabled())
+ log.info("Established outgoing communication connection [locAddr=" + ses.localAddress() +
+ ", rmtAddr=" + ses.remoteAddress() + ']');
}
}
@Override public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
ConnectionKey connId = ses.meta(CONN_IDX_META);
+ if (connId == CONN_CHECK_DUMMY_KEY)
+ return;
+
if (connId != null) {
UUID id = connId.nodeId();
@@ -691,7 +690,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ConnectionKey connKey = ses.meta(CONN_IDX_META);
if (connKey == null) {
- assert ses.accepted() : msg;
+ assert ses.accepted() : ses;
if (!connectGate.tryEnter()) {
if (log.isDebugEnabled())
@@ -714,9 +713,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
else {
- metricsLsnr.onMessageReceived(msg, connKey.nodeId());
-
if (msg instanceof RecoveryLastReceivedMessage) {
+ metricsLsnr.onMessageReceived(msg, connKey.nodeId());
+
GridNioRecoveryDescriptor recovery = ses.outRecoveryDescriptor();
if (recovery != null) {
@@ -729,9 +728,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
recovery.ackReceived(msg0.received());
-
- return;
}
+
+ return;
}
else {
GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
@@ -754,9 +753,11 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
else if (connKey == CONN_CHECK_DUMMY_KEY) {
assert msg instanceof NodeIdMessage : msg;
- TcpCommunicationConnectionCheckFuture fut = ses.meta(SES_FUT_META);
+ TcpCommunicationNodeConnectionCheckFuture fut = ses.meta(SES_FUT_META);
+
+ assert fut != null : msg;
- fut.onConnected(U.bytesToUuid(((NodeIdMessage) msg).nodeIdBytes, 0));
+ fut.onConnected(U.bytesToUuid(((NodeIdMessage)msg).nodeIdBytes(), 0));
nioSrvr.closeFromWorkerThread(ses);
@@ -764,6 +765,8 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
}
+ metricsLsnr.onMessageReceived(msg, connKey.nodeId());
+
IgniteRunnable c;
if (msgQueueLimit > 0) {
@@ -2592,18 +2595,17 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
sendMessage0(node, msg, null);
}
+ /** {@inheritDoc} */
public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
- ClusterNode node = nodes.get(0);
-
- try {
- Collection<InetSocketAddress> addrs = nodeAddresses(node);
+ TcpCommunicationConnectionCheckFuture fut = new TcpCommunicationConnectionCheckFuture(
+ this,
+ log.getLogger(TcpCommunicationConnectionCheckFuture.class),
+ nioSrvr,
+ nodes);
- }
- catch (Exception e) {
- throw new IgniteSpiException(e);
- }
+ fut.init(failureDetectionTimeoutEnabled() ? failureDetectionTimeout() : connTimeout);
- return null;
+ return new IgniteFutureImpl<>(fut);
}
/**
@@ -3028,7 +3030,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ConnectionKey id = ses.meta(CONN_IDX_META);
if (id != null) {
- ClusterNode node = getSpiContext().node(id.nodeId);
+ ClusterNode node = getSpiContext().node(id.nodeId());
if (node != null && node.isClient()) {
String msg = "Client node outbound message queue size exceeded slowClientQueueLimit, " +
@@ -3049,9 +3051,20 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/**
* @param node Node.
* @return Node addresses.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Collection<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
+ return nodeAddresses(node, filterReachableAddresses);
+ }
+
+ /**
+ * @param node Node.
+ * @param filterReachableAddresses Filter addresses flag.
+ * @return Node addresses.
* @throws IgniteCheckedException If node does not have addresses.
*/
- private LinkedHashSet<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
+ public Collection<InetSocketAddress> nodeAddresses(ClusterNode node, boolean filterReachableAddresses)
+ throws IgniteCheckedException {
Collection<String> rmtAddrs0 = node.attribute(createSpiAttributeName(ATTR_ADDRS));
Collection<String> rmtHostNames0 = node.attribute(createSpiAttributeName(ATTR_HOST_NAMES));
Integer boundPort = node.attribute(createSpiAttributeName(ATTR_PORT));
@@ -3132,7 +3145,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @throws IgniteCheckedException If failed.
*/
protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
- LinkedHashSet<InetSocketAddress> addrs = nodeAddresses(node);
+ Collection<InetSocketAddress> addrs = nodeAddresses(node);
GridCommunicationClient client = null;
IgniteCheckedException errs = null;
@@ -4629,77 +4642,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
/**
*
*/
- private static class ConnectionKey {
- /** */
- private final UUID nodeId;
-
- /** */
- private final int idx;
-
- /** */
- private final long connCnt;
-
- /**
- * @param nodeId Node ID.
- * @param idx Connection index.
- * @param connCnt Connection counter (set only for incoming connections).
- */
- ConnectionKey(UUID nodeId, int idx, long connCnt) {
- this.nodeId = nodeId;
- this.idx = idx;
- this.connCnt = connCnt;
- }
-
- /**
- * @return Connection counter.
- */
- long connectCount() {
- return connCnt;
- }
-
- /**
- * @return Node ID.
- */
- UUID nodeId() {
- return nodeId;
- }
-
- /**
- * @return Connection index.
- */
- int connectionIndex() {
- return idx;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- ConnectionKey key = (ConnectionKey) o;
-
- return idx == key.idx && nodeId.equals(key.nodeId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = nodeId.hashCode();
- res = 31 * res + idx;
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(ConnectionKey.class, this);
- }
- }
-
- /**
- *
- */
interface ConnectionPolicy {
/**
* @return Thread connection index.
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed492a4f/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
new file mode 100644
index 0000000..6716446
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class ConnectionKey {
+ /** */
+ private final UUID nodeId;
+
+ /** */
+ private final int idx;
+
+ /** */
+ private final long connCnt;
+
+ /**
+ * @param nodeId Node ID.
+ * @param idx Connection index.
+ * @param connCnt Connection counter (set only for incoming connections).
+ */
+ public ConnectionKey(UUID nodeId, int idx, long connCnt) {
+ this.nodeId = nodeId;
+ this.idx = idx;
+ this.connCnt = connCnt;
+ }
+
+ /**
+ * @return Connection counter.
+ */
+ public long connectCount() {
+ return connCnt;
+ }
+
+ /**
+ * @return Node ID.
+ */
+ public UUID nodeId() {
+ return nodeId;
+ }
+
+ /**
+ * @return Connection index.
+ */
+ public int connectionIndex() {
+ return idx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ ConnectionKey key = (ConnectionKey) o;
+
+ return idx == key.idx && nodeId.equals(key.nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = nodeId.hashCode();
+ res = 31 * res + idx;
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(ConnectionKey.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed492a4f/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
index e08ea13..99e1eca 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
@@ -17,84 +17,445 @@
package org.apache.ignite.spi.communication.tcp.internal;
-import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.channels.SocketChannel;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.GridLeanMap;
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.nio.GridNioServer;
import org.apache.ignite.internal.util.nio.GridNioSession;
+import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jetbrains.annotations.Nullable;
/**
*
*/
-public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<Boolean> {
+public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<BitSet> implements IgniteSpiTimeoutObject {
+ /** Session future. */
+ public static final int SES_FUT_META = GridNioSessionMetaKey.nextUniqueKey();
+
+ /** */
+ public static final ConnectionKey CONN_CHECK_DUMMY_KEY = new ConnectionKey(null, -1, -1);
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<SingleAddressConnectFuture> connFutDoneUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(SingleAddressConnectFuture.class, "done");
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<MultipleAddressesConnectFuture> connResCntUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(MultipleAddressesConnectFuture.class, "resCnt");
+
+ /** */
+ private final AtomicInteger resCntr = new AtomicInteger();
+
/** */
- private final UUID nodeId;
+ private final List<ClusterNode> nodes;
+
+ /** */
+ private volatile ConnectFuture[] futs;
/** */
private final GridNioServer nioSrvr;
/** */
- private Map<Integer, Object> sesMeta;
+ private final TcpCommunicationSpi spi;
+
+ /** */
+ private final IgniteUuid timeoutObjId = IgniteUuid.randomUuid();
+
+ /** */
+ private final BitSet resBitSet;
+
+ /** */
+ private long endTime;
/** */
- private SocketChannel ch;
+ private final IgniteLogger log;
/**
- * @param nodeId Remote note ID.
+ * @param spi SPI instance.
+ * @param log Logger.
+ * @param nioSrvr NIO server.
+ * @param nodes Nodes to check.
*/
- public TcpCommunicationConnectionCheckFuture(GridNioServer nioSrvr, UUID nodeId) {
+ public TcpCommunicationConnectionCheckFuture(TcpCommunicationSpi spi,
+ IgniteLogger log,
+ GridNioServer nioSrvr,
+ List<ClusterNode> nodes)
+ {
+ this.spi = spi;
+ this.log = log;
this.nioSrvr = nioSrvr;
- this.nodeId = nodeId;
+ this.nodes = nodes;
+
+ resBitSet = new BitSet(nodes.size());
}
/**
- * @param addr
- * @throws IOException
+ * @param timeout Connect timeout.
*/
- public void init(InetSocketAddress addr) throws IOException {
- ch = SocketChannel.open();
+ public void init(long timeout) {
+ ConnectFuture[] futs = new ConnectFuture[nodes.size()];
+
+ UUID locId = spi.getSpiContext().localNode().id();
+
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+
+ if (!node.id().equals(locId)) {
+ if (spi.getSpiContext().node(node.id()) == null) {
+ receivedConnectionStatus(i, false);
+
+ continue;
+ }
+
+ Collection<InetSocketAddress> addrs;
+
+ try {
+ addrs = spi.nodeAddresses(node, false);
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to get node addresses: " + node, e);
- ch.configureBlocking(false);
+ receivedConnectionStatus(i, false);
- ch.socket().setTcpNoDelay(true);
- ch.socket().setKeepAlive(false);
+ continue;
+ }
- boolean connect = ch.connect(addr);
+ if (addrs.size() == 1) {
+ SingleAddressConnectFuture fut = new SingleAddressConnectFuture(i);
- if (!connect) {
- sesMeta = new GridLeanMap<>(2);
+ fut.init(addrs.iterator().next());
- sesMeta.put(TcpCommunicationSpi.CONN_IDX_META, TcpCommunicationSpi.CONN_CHECK_DUMMY_KEY);
- sesMeta.put(TcpCommunicationSpi.SES_FUT_META, this);
+ futs[i] = fut;
+ }
+ else {
+ MultipleAddressesConnectFuture fut = new MultipleAddressesConnectFuture(i);
- nioSrvr.createSession(ch, sesMeta, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
- @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
- if (fut.error() != null)
- onDone(false);
+ fut.init(addrs);
}
- });
+ }
+ else
+ receivedConnectionStatus(i, true);
+ }
+
+ this.futs = futs;
+
+ if (!isDone()) {
+ endTime = System.currentTimeMillis() - timeout;
+
+ spi.getSpiContext().addTimeoutObject(this);
}
}
/**
+ * @param idx Node index.
+ * @param res Success flag.
+ */
+ private void receivedConnectionStatus(int idx, boolean res) {
+ assert resCntr.get() < nodes.size();
+
+ synchronized (resBitSet) {
+ resBitSet.set(idx, res);
+ }
+
+ if (resCntr.incrementAndGet() == nodes.size())
+ onDone(resBitSet);
+ }
+
+ /**
+ * @param nodeIdx Node index.
+ * @return Node ID.
+ */
+ private UUID nodeId(int nodeIdx) {
+ return nodes.get(nodeIdx).id();
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid id() {
+ return timeoutObjId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return endTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ if (!isDone())
+ return;
+
+ ConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onTimeout();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onDone(@Nullable BitSet res, @Nullable Throwable err) {
+ if (super.onDone(res, err)) {
+ spi.getSpiContext().removeTimeoutObject(this);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
*
*/
- public void onTimeout() {
- if (super.onDone(false))
- nioSrvr.cancelConnect(ch, sesMeta);
+ private interface ConnectFuture {
+ /**
+ *
+ */
+ void onTimeout();
}
/**
- * @param rmtNodeId
+ *
*/
- public void onConnected(UUID rmtNodeId) {
- onDone(nodeId.equals(rmtNodeId));
+ private class SingleAddressConnectFuture implements TcpCommunicationNodeConnectionCheckFuture, ConnectFuture {
+ /** */
+ final int nodeIdx;
+
+ /** */
+ volatile int done;
+
+ /** */
+ Map<Integer, Object> sesMeta;
+
+ /** */
+ private SocketChannel ch;
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ SingleAddressConnectFuture(int nodeIdx) {
+ this.nodeIdx = nodeIdx;
+ }
+
+ /**
+ * @param addr Node address.
+ */
+ public void init(InetSocketAddress addr) {
+ boolean connect;
+
+ try {
+ ch = SocketChannel.open();
+
+ ch.configureBlocking(false);
+
+ ch.socket().setTcpNoDelay(true);
+ ch.socket().setKeepAlive(false);
+
+ connect = ch.connect(addr);
+ }
+ catch (Exception e) {
+ finish(false);
+
+ return;
+ }
+
+ if (!connect) {
+ sesMeta = new GridLeanMap<>(3);
+
+ // Set dummy key to identify connection-check outgoing connection.
+ sesMeta.put(TcpCommunicationSpi.CONN_IDX_META, CONN_CHECK_DUMMY_KEY);
+ sesMeta.put(SES_FUT_META, this);
+
+ nioSrvr.createSession(ch, sesMeta, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
+ @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
+ if (fut.error() != null)
+ finish(false);
+ }
+ });
+ }
+ }
+
+ /**
+ *
+ */
+ void cancel() {
+ if (finish(false))
+ nioSrvr.cancelConnect(ch, sesMeta);
+ }
+
+ /** {@inheritDoc} */
+ public void onTimeout() {
+ cancel();
+ }
+
+ /** {@inheritDoc} */
+ public void onConnected(UUID rmtNodeId) {
+ finish(nodeId(nodeIdx).equals(rmtNodeId));
+ }
+
+ /**
+ * @param res Result.
+ * @return {@code True} if result was set by this call.
+ */
+ boolean finish(boolean res) {
+ if (connFutDoneUpdater.compareAndSet(this, 0, 1)) {
+ onStatusReceived(res);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param res Result.
+ */
+ void onStatusReceived(boolean res) {
+ receivedConnectionStatus(nodeIdx, res);
+ }
+ }
+
+ /**
+ *
+ */
+ private class MultipleAddressesConnectFuture implements ConnectFuture {
+ /** */
+ volatile int resCnt;
+
+ /** */
+ volatile SingleAddressConnectFuture[] futs;
+
+ /** */
+ final int nodeIdx;
+
+ /**
+ * @param nodeIdx Node index.
+ */
+ MultipleAddressesConnectFuture(int nodeIdx) {
+ this.nodeIdx = nodeIdx;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ for (int i = 0; i < futs.length; i++) {
+ ConnectFuture fut = futs[i];
+
+ if (fut != null)
+ fut.onTimeout();
+ }
+ }
+
+ /**
+ * @param addrs Node addresses.
+ */
+ void init(Collection<InetSocketAddress> addrs) {
+ SingleAddressConnectFuture[] futs = new SingleAddressConnectFuture[addrs.size()];
+
+ int idx = 0;
+
+ for (InetSocketAddress addr : addrs) {
+ SingleAddressConnectFuture fut = new SingleAddressConnectFuture(nodeIdx) {
+ @Override void onStatusReceived(boolean res) {
+ receivedAddressStatus(res);
+ }
+ };
+
+ fut.init(addr);
+
+ futs[idx++] = fut;
+
+ if (done())
+ return;
+ }
+
+ this.futs = futs;
+
+ // Close race.
+ if (done())
+ cancelFutures();
+ }
+
+ /**
+ * @return {@code True}
+ */
+ private boolean done() {
+ int resCnt0 = resCnt;
+
+ return resCnt0 == Integer.MAX_VALUE || resCnt0 == futs.length;
+ }
+
+ /**
+ *
+ */
+ private void cancelFutures() {
+ SingleAddressConnectFuture[] futs = this.futs;
+
+ if (futs != null) {
+ for (int i = 0; i < futs.length; i++) {
+ SingleAddressConnectFuture fut = futs[i];
+
+ fut.cancel();
+ }
+ }
+ }
+
+ /**
+ * @param res Result.
+ */
+ void receivedAddressStatus(boolean res) {
+ if (res) {
+ for (;;) {
+ int resCnt0 = resCnt;
+
+ if (resCnt0 == Integer.MAX_VALUE)
+ return;
+
+ if (connResCntUpdater.compareAndSet(this, resCnt0, Integer.MAX_VALUE)) {
+ receivedConnectionStatus(nodeIdx, true);
+
+ cancelFutures(); // Cancel others connects if they are still in progress.
+
+ return;
+ }
+ }
+ }
+ else {
+ for (;;) {
+ int resCnt0 = resCnt;
+
+ if (resCnt0 == Integer.MAX_VALUE)
+ return;
+
+ int resCnt1 = resCnt0 + 1;
+
+ if (connResCntUpdater.compareAndSet(this, resCnt0, resCnt1)) {
+ if (resCnt1 == futs.length)
+ receivedConnectionStatus(nodeIdx, false);
+
+ return;
+ }
+ }
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed492a4f/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
new file mode 100644
index 0000000..c034782
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.util.UUID;
+
+/**
+ *
+ */
+public interface TcpCommunicationNodeConnectionCheckFuture {
+ /**
+ * @param nodeId Remote node ID.
+ */
+ public void onConnected(UUID nodeId);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/ed492a4f/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
index caa2d87..e89a4c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
@@ -18,20 +18,23 @@
package org.apache.ignite.spi.communication.tcp;
import java.util.ArrayList;
-import java.util.HashSet;
+import java.util.BitSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.util.nio.GridCommunicationClient;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.IgniteSpiAdapter;
import org.apache.ignite.spi.communication.CommunicationSpi;
import org.apache.ignite.spi.communication.GridAbstractCommunicationSelfTest;
-import org.apache.ignite.spi.communication.GridTestMessage;
import org.apache.ignite.testframework.GridTestUtils;
/**
@@ -39,7 +42,7 @@ import org.apache.ignite.testframework.GridTestUtils;
*/
abstract class GridTcpCommunicationSpiAbstractTest extends GridAbstractCommunicationSelfTest<CommunicationSpi> {
/** */
- private static final int SPI_COUNT = 2;
+ private static final int SPI_COUNT = 3;
/** */
public static final int IDLE_CONN_TIMEOUT = 2000;
@@ -92,33 +95,65 @@ abstract class GridTcpCommunicationSpiAbstractTest extends GridAbstractCommunica
}
}
- public void testConnectionCheck() {
- for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
- UUID id = entry.getKey();
+ /**
+ *
+ */
+ public void testCheckConnection1() {
+ for (int i = 0; i < 100; i++) {
+ for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+ TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
- TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+ List<ClusterNode> checkNodes = new ArrayList<>(nodes);
- List<ClusterNode> checkNodes = new ArrayList<>();
+ assert checkNodes.size() > 1;
- for (ClusterNode node : nodes) {
- if (!id.equals(node.id()))
- checkNodes.add(node);
+ IgniteFuture<BitSet> fut = spi.checkConnection(checkNodes);
+
+ BitSet res = fut.get();
+
+ for (int n = 0; n < checkNodes.size(); n++)
+ assertTrue(res.get(n));
}
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCheckConnection2() throws Exception {
+ final int THREADS = spis.size();
+
+ final CyclicBarrier b = new CyclicBarrier(THREADS);
+
+ List<IgniteInternalFuture> futs = new ArrayList<>();
- spi.checkConnection(checkNodes);
-
- break;
-// for (ClusterNode node : nodes) {
-// synchronized (mux) {
-// if (!msgDestMap.containsKey(entry.getKey()))
-// msgDestMap.put(entry.getKey(), new HashSet<UUID>());
-//
-// msgDestMap.get(entry.getKey()).add(node.id());
-// }
-//
-// entry.getValue().sendMessage(node, new GridTestMessage(entry.getKey(), msgId++, 0));
-// }
+ for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+ final TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+
+ futs.add(GridTestUtils.runAsync(new Callable() {
+ @Override public Object call() throws Exception {
+ List<ClusterNode> checkNodes = new ArrayList<>(nodes);
+
+ assert checkNodes.size() > 1;
+
+ b.await();
+
+ for (int i = 0; i < 100; i++) {
+ IgniteFuture<BitSet> fut = spi.checkConnection(checkNodes);
+
+ BitSet res = fut.get();
+
+ for (int n = 0; n < checkNodes.size(); n++)
+ assertTrue(res.get(n));
+ }
+
+ return null;
+ }
+ }));
}
+
+ for (IgniteInternalFuture f : futs)
+ f.get();
}
/** {@inheritDoc} */
[26/50] [abbrv] ignite git commit: Minor: moved inner
TcpCommunicationSpi messages to top-level classes.
Posted by sb...@apache.org.
Minor: moved inner TcpCommunicationSpi messages to top-level classes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c576d5a0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c576d5a0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c576d5a0
Branch: refs/heads/ignite-zk
Commit: c576d5a0ab1bbaab4a1b23a2c03a21ef2a7b610c
Parents: 061ec6a
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 11:28:18 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 11:28:18 2017 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 14 +-
.../spi/communication/tcp/HandshakeMessage.java | 156 ++++++++
.../communication/tcp/HandshakeMessage2.java | 95 +++++
.../spi/communication/tcp/NodeIdMessage.java | 115 ++++++
.../tcp/RecoveryLastReceivedMessage.java | 113 ++++++
.../communication/tcp/TcpCommunicationSpi.java | 391 +------------------
6 files changed, 492 insertions(+), 392 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c576d5a0/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 97e06bf..791dd91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -137,11 +137,11 @@ import org.apache.ignite.internal.processors.igfs.IgfsFragmentizerResponse;
import org.apache.ignite.internal.processors.igfs.IgfsSyncMessage;
import org.apache.ignite.internal.processors.marshaller.MissingMappingRequestMessage;
import org.apache.ignite.internal.processors.marshaller.MissingMappingResponseMessage;
-import org.apache.ignite.internal.processors.query.schema.message.SchemaOperationStatusMessage;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryFailResponse;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageRequest;
import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryNextPageResponse;
+import org.apache.ignite.internal.processors.query.schema.message.SchemaOperationStatusMessage;
import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest;
import org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultResponse;
import org.apache.ignite.internal.util.GridByteArrayList;
@@ -153,6 +153,10 @@ import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageFactory;
import org.apache.ignite.spi.collision.jobstealing.JobStealingRequest;
+import org.apache.ignite.spi.communication.tcp.HandshakeMessage;
+import org.apache.ignite.spi.communication.tcp.HandshakeMessage2;
+import org.apache.ignite.spi.communication.tcp.NodeIdMessage;
+import org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.jsr166.ConcurrentHashMap8;
@@ -221,7 +225,7 @@ public class GridIoMessageFactory implements MessageFactory {
break;
case -44:
- msg = new TcpCommunicationSpi.HandshakeMessage2();
+ msg = new HandshakeMessage2();
break;
@@ -291,17 +295,17 @@ public class GridIoMessageFactory implements MessageFactory {
break;
case TcpCommunicationSpi.NODE_ID_MSG_TYPE:
- msg = new TcpCommunicationSpi.NodeIdMessage();
+ msg = new NodeIdMessage();
break;
case TcpCommunicationSpi.RECOVERY_LAST_ID_MSG_TYPE:
- msg = new TcpCommunicationSpi.RecoveryLastReceivedMessage();
+ msg = new RecoveryLastReceivedMessage();
break;
case TcpCommunicationSpi.HANDSHAKE_MSG_TYPE:
- msg = new TcpCommunicationSpi.HandshakeMessage();
+ msg = new HandshakeMessage();
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c576d5a0/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java
new file mode 100644
index 0000000..00e8e46
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Handshake message.
+ */
+public class HandshakeMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Message body size in bytes. */
+ private static final int MESSAGE_SIZE = 32;
+
+ /** Full message size (with message type) in bytes. */
+ public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
+
+ /** */
+ private UUID nodeId;
+
+ /** */
+ private long rcvCnt;
+
+ /** */
+ private long connectCnt;
+
+ /**
+ * Default constructor required by {@link Message}.
+ */
+ public HandshakeMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param connectCnt Connect count.
+ * @param rcvCnt Number of received messages.
+ */
+ public HandshakeMessage(UUID nodeId, long connectCnt, long rcvCnt) {
+ assert nodeId != null;
+ assert rcvCnt >= 0 : rcvCnt;
+
+ this.nodeId = nodeId;
+ this.connectCnt = connectCnt;
+ this.rcvCnt = rcvCnt;
+ }
+
+ /**
+ * @return Connection index.
+ */
+ public int connectionIndex() {
+ return 0;
+ }
+
+ /**
+ * @return Connect count.
+ */
+ public long connectCount() {
+ return connectCnt;
+ }
+
+ /**
+ * @return Number of received messages.
+ */
+ public long received() {
+ return rcvCnt;
+ }
+
+ /**
+ * @return Node ID.
+ */
+ public UUID nodeId() {
+ return nodeId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ if (buf.remaining() < MESSAGE_FULL_SIZE)
+ return false;
+
+ TcpCommunicationSpi.writeMessageType(buf, directType());
+
+ byte[] bytes = U.uuidToBytes(nodeId);
+
+ assert bytes.length == 16 : bytes.length;
+
+ buf.put(bytes);
+
+ buf.putLong(rcvCnt);
+
+ buf.putLong(connectCnt);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (buf.remaining() < MESSAGE_SIZE)
+ return false;
+
+ byte[] nodeIdBytes = new byte[NodeIdMessage.MESSAGE_SIZE];
+
+ buf.get(nodeIdBytes);
+
+ nodeId = U.bytesToUuid(nodeIdBytes, 0);
+
+ rcvCnt = buf.getLong();
+
+ connectCnt = buf.getLong();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TcpCommunicationSpi.HANDSHAKE_MSG_TYPE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HandshakeMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c576d5a0/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java
new file mode 100644
index 0000000..1e8fdd9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Updated handshake message.
+ */
+public class HandshakeMessage2 extends HandshakeMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private int connIdx;
+
+ /**
+ *
+ */
+ public HandshakeMessage2() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param connectCnt Connect count.
+ * @param rcvCnt Number of received messages.
+ * @param connIdx Connection index.
+ */
+ HandshakeMessage2(UUID nodeId, long connectCnt, long rcvCnt, int connIdx) {
+ super(nodeId, connectCnt, rcvCnt);
+
+ this.connIdx = connIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return -44;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int connectionIndex() {
+ return connIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (buf.remaining() < 4)
+ return false;
+
+ buf.putInt(connIdx);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ if (buf.remaining() < 4)
+ return false;
+
+ connIdx = buf.getInt();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HandshakeMessage2.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c576d5a0/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java
new file mode 100644
index 0000000..d05b7ff
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Node ID message.
+ */
+public class NodeIdMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Message body size (with message type) in bytes. */
+ static final int MESSAGE_SIZE = 16;
+
+ /** Full message size (with message type) in bytes. */
+ public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
+
+ /** */
+ byte[] nodeIdBytes;
+
+ /** */
+ byte[] nodeIdBytesWithType;
+
+ /** */
+ public NodeIdMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ */
+ NodeIdMessage(UUID nodeId) {
+ assert nodeId != null;
+
+ nodeIdBytes = U.uuidToBytes(nodeId);
+
+ assert nodeIdBytes.length == MESSAGE_SIZE : "Node ID size must be " + MESSAGE_SIZE;
+
+ nodeIdBytesWithType = new byte[MESSAGE_FULL_SIZE];
+
+ nodeIdBytesWithType[0] = (byte)(TcpCommunicationSpi.NODE_ID_MSG_TYPE & 0xFF);
+ nodeIdBytesWithType[1] = (byte)((TcpCommunicationSpi.NODE_ID_MSG_TYPE >> 8) & 0xFF);
+
+ System.arraycopy(nodeIdBytes, 0, nodeIdBytesWithType, 2, nodeIdBytes.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ assert nodeIdBytes.length == MESSAGE_SIZE;
+
+ if (buf.remaining() < MESSAGE_FULL_SIZE)
+ return false;
+
+ TcpCommunicationSpi.writeMessageType(buf, directType());
+
+ buf.put(nodeIdBytes);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (buf.remaining() < MESSAGE_SIZE)
+ return false;
+
+ nodeIdBytes = new byte[MESSAGE_SIZE];
+
+ buf.get(nodeIdBytes);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TcpCommunicationSpi.NODE_ID_MSG_TYPE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(NodeIdMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c576d5a0/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java
new file mode 100644
index 0000000..5460084
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Recovery acknowledgment message.
+ */
+public class RecoveryLastReceivedMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ static final long ALREADY_CONNECTED = -1;
+
+ /** */
+ static final long NODE_STOPPING = -2;
+
+ /** Need wait. */
+ static final long NEED_WAIT = -3;
+
+ /** Message body size in bytes. */
+ private static final int MESSAGE_SIZE = 8;
+
+ /** Full message size (with message type) in bytes. */
+ public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
+
+ /** */
+ private long rcvCnt;
+
+ /**
+ * Default constructor required by {@link Message}.
+ */
+ public RecoveryLastReceivedMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param rcvCnt Number of received messages.
+ */
+ public RecoveryLastReceivedMessage(long rcvCnt) {
+ this.rcvCnt = rcvCnt;
+ }
+
+ /**
+ * @return Number of received messages.
+ */
+ public long received() {
+ return rcvCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ if (buf.remaining() < MESSAGE_FULL_SIZE)
+ return false;
+
+ TcpCommunicationSpi.writeMessageType(buf, directType());
+
+ buf.putLong(rcvCnt);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (buf.remaining() < MESSAGE_SIZE)
+ return false;
+
+ rcvCnt = buf.getLong();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TcpCommunicationSpi.RECOVERY_LAST_ID_MSG_TYPE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(RecoveryLastReceivedMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c576d5a0/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 9290f24..1f0061f 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -141,9 +141,9 @@ import org.jsr166.ConcurrentLinkedDeque8;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
-import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
-import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.RecoveryLastReceivedMessage.NEED_WAIT;
-import static org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.RecoveryLastReceivedMessage.NODE_STOPPING;
+import static org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
+import static org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage.NEED_WAIT;
+import static org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage.NODE_STOPPING;
/**
* <tt>TcpCommunicationSpi</tt> is default communication SPI which uses
@@ -3923,7 +3923,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @param buf Byte buffer.
* @param type Message type.
*/
- private static void writeMessageType(ByteBuffer buf, short type) {
+ static void writeMessageType(ByteBuffer buf, short type) {
buf.put((byte)(type & 0xFF));
buf.put((byte)((type >> 8) & 0xFF));
}
@@ -4440,389 +4440,6 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
}
/**
- * Handshake message.
- */
- @SuppressWarnings("PublicInnerClass")
- public static class HandshakeMessage implements Message {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Message body size in bytes. */
- private static final int MESSAGE_SIZE = 32;
-
- /** Full message size (with message type) in bytes. */
- public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
-
- /** */
- private UUID nodeId;
-
- /** */
- private long rcvCnt;
-
- /** */
- private long connectCnt;
-
- /**
- * Default constructor required by {@link Message}.
- */
- public HandshakeMessage() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- * @param connectCnt Connect count.
- * @param rcvCnt Number of received messages.
- */
- public HandshakeMessage(UUID nodeId, long connectCnt, long rcvCnt) {
- assert nodeId != null;
- assert rcvCnt >= 0 : rcvCnt;
-
- this.nodeId = nodeId;
- this.connectCnt = connectCnt;
- this.rcvCnt = rcvCnt;
- }
-
- /**
- * @return Connection index.
- */
- public int connectionIndex() {
- return 0;
- }
-
- /**
- * @return Connect count.
- */
- public long connectCount() {
- return connectCnt;
- }
-
- /**
- * @return Number of received messages.
- */
- public long received() {
- return rcvCnt;
- }
-
- /**
- * @return Node ID.
- */
- public UUID nodeId() {
- return nodeId;
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- if (buf.remaining() < MESSAGE_FULL_SIZE)
- return false;
-
- writeMessageType(buf, directType());
-
- byte[] bytes = U.uuidToBytes(nodeId);
-
- assert bytes.length == 16 : bytes.length;
-
- buf.put(bytes);
-
- buf.putLong(rcvCnt);
-
- buf.putLong(connectCnt);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (buf.remaining() < MESSAGE_SIZE)
- return false;
-
- byte[] nodeIdBytes = new byte[NodeIdMessage.MESSAGE_SIZE];
-
- buf.get(nodeIdBytes);
-
- nodeId = U.bytesToUuid(nodeIdBytes, 0);
-
- rcvCnt = buf.getLong();
-
- connectCnt = buf.getLong();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return HANDSHAKE_MSG_TYPE;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HandshakeMessage.class, this);
- }
- }
-
- /**
- * Updated handshake message.
- */
- @SuppressWarnings("PublicInnerClass")
- public static class HandshakeMessage2 extends HandshakeMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private int connIdx;
-
- /**
- *
- */
- public HandshakeMessage2() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- * @param connectCnt Connect count.
- * @param rcvCnt Number of received messages.
- * @param connIdx Connection index.
- */
- HandshakeMessage2(UUID nodeId, long connectCnt, long rcvCnt, int connIdx) {
- super(nodeId, connectCnt, rcvCnt);
-
- this.connIdx = connIdx;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return -44;
- }
-
- /** {@inheritDoc} */
- @Override public int connectionIndex() {
- return connIdx;
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- if (!super.writeTo(buf, writer))
- return false;
-
- if (buf.remaining() < 4)
- return false;
-
- buf.putInt(connIdx);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (!super.readFrom(buf, reader))
- return false;
-
- if (buf.remaining() < 4)
- return false;
-
- connIdx = buf.getInt();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HandshakeMessage2.class, this);
- }
- }
-
- /**
- * Recovery acknowledgment message.
- */
- @SuppressWarnings("PublicInnerClass")
- public static class RecoveryLastReceivedMessage implements Message {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- static final long ALREADY_CONNECTED = -1;
-
- /** */
- static final long NODE_STOPPING = -2;
-
- /** Need wait. */
- static final long NEED_WAIT = -3;
-
- /** Message body size in bytes. */
- private static final int MESSAGE_SIZE = 8;
-
- /** Full message size (with message type) in bytes. */
- public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
-
- /** */
- private long rcvCnt;
-
- /**
- * Default constructor required by {@link Message}.
- */
- public RecoveryLastReceivedMessage() {
- // No-op.
- }
-
- /**
- * @param rcvCnt Number of received messages.
- */
- public RecoveryLastReceivedMessage(long rcvCnt) {
- this.rcvCnt = rcvCnt;
- }
-
- /**
- * @return Number of received messages.
- */
- public long received() {
- return rcvCnt;
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- if (buf.remaining() < MESSAGE_FULL_SIZE)
- return false;
-
- writeMessageType(buf, directType());
-
- buf.putLong(rcvCnt);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (buf.remaining() < MESSAGE_SIZE)
- return false;
-
- rcvCnt = buf.getLong();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return RECOVERY_LAST_ID_MSG_TYPE;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(RecoveryLastReceivedMessage.class, this);
- }
- }
-
- /**
- * Node ID message.
- */
- @SuppressWarnings("PublicInnerClass")
- public static class NodeIdMessage implements Message {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Message body size (with message type) in bytes. */
- private static final int MESSAGE_SIZE = 16;
-
- /** Full message size (with message type) in bytes. */
- public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
-
- /** */
- private byte[] nodeIdBytes;
-
- /** */
- private byte[] nodeIdBytesWithType;
-
- /** */
- public NodeIdMessage() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- */
- private NodeIdMessage(UUID nodeId) {
- assert nodeId != null;
-
- nodeIdBytes = U.uuidToBytes(nodeId);
-
- assert nodeIdBytes.length == MESSAGE_SIZE : "Node ID size must be " + MESSAGE_SIZE;
-
- nodeIdBytesWithType = new byte[MESSAGE_FULL_SIZE];
-
- nodeIdBytesWithType[0] = (byte)(NODE_ID_MSG_TYPE & 0xFF);
- nodeIdBytesWithType[1] = (byte)((NODE_ID_MSG_TYPE >> 8) & 0xFF);
-
- System.arraycopy(nodeIdBytes, 0, nodeIdBytesWithType, 2, nodeIdBytes.length);
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- assert nodeIdBytes.length == MESSAGE_SIZE;
-
- if (buf.remaining() < MESSAGE_FULL_SIZE)
- return false;
-
- writeMessageType(buf, directType());
-
- buf.put(nodeIdBytes);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (buf.remaining() < MESSAGE_SIZE)
- return false;
-
- nodeIdBytes = new byte[MESSAGE_SIZE];
-
- buf.get(nodeIdBytes);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return NODE_ID_MSG_TYPE;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(NodeIdMessage.class, this);
- }
- }
-
- /**
*
*/
private class ConnectGateway {
[04/50] [abbrv] ignite git commit: IGNITE-7166: SQL: fixed affinity
nodes processing for REPLICATED caches. This closes #3202.
Posted by sb...@apache.org.
IGNITE-7166: SQL: fixed affinity nodes processing for REPLICATED caches. This closes #3202.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e3bbc984
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e3bbc984
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e3bbc984
Branch: refs/heads/ignite-zk
Commit: e3bbc984c1acd65ad9d822ad6fb634221d1a5a2d
Parents: 7cf049e
Author: devozerov <pp...@gmail.com>
Authored: Tue Dec 12 20:07:07 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Tue Dec 12 20:07:07 2017 +0300
----------------------------------------------------------------------
.../processors/affinity/AffinityAssignment.java | 5 +
.../affinity/GridAffinityAssignment.java | 45 ++++--
.../affinity/HistoryAffinityAssignment.java | 15 ++
.../h2/twostep/GridReduceQueryExecutor.java | 44 +++---
...ldsQueryJoinNoPrimaryPartitionsSelfTest.java | 151 +++++++++++++++++++
.../IgniteCacheQuerySelfTestSuite.java | 2 +
6 files changed, 223 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3bbc984/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
index 06207d3..f78ab60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityAssignment.java
@@ -66,6 +66,11 @@ public interface AffinityAssignment {
public HashSet<UUID> getIds(int part);
/**
+ * @return Nodes having parimary and backup assignments.
+ */
+ public Set<ClusterNode> nodes();
+
+ /**
* @return Nodes having primary partitions assignments.
*/
public Set<ClusterNode> primaryPartitionNodes();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3bbc984/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
index 35130a3..6da6aaa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
@@ -32,6 +32,7 @@ import org.apache.ignite.internal.util.typedef.internal.S;
/**
* Cached affinity calculations.
*/
+@SuppressWarnings("ForLoopReplaceableByForEach")
public class GridAffinityAssignment implements AffinityAssignment, Serializable {
/** */
private static final long serialVersionUID = 0L;
@@ -51,6 +52,9 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
/** Assignment node IDs */
private transient volatile List<HashSet<UUID>> assignmentIds;
+ /** Nodes having primary or backup partition assignments. */
+ private transient volatile Set<ClusterNode> nodes;
+
/** Nodes having primary partitions assignments. */
private transient volatile Set<ClusterNode> primaryPartsNodes;
@@ -182,29 +186,44 @@ public class GridAffinityAssignment implements AffinityAssignment, Serializable
return assignmentIds0.get(part);
}
- /**
- * @return Nodes having primary partitions assignments.
- */
- @SuppressWarnings("ForLoopReplaceableByForEach")
- public Set<ClusterNode> primaryPartitionNodes() {
- Set<ClusterNode> primaryPartsNodes0 = primaryPartsNodes;
+ /** {@inheritDoc} */
+ @Override public Set<ClusterNode> nodes() {
+ Set<ClusterNode> res = nodes;
- if (primaryPartsNodes0 == null) {
- int parts = assignment.size();
+ if (res == null) {
+ res = new HashSet<>();
+
+ for (int p = 0; p < assignment.size(); p++) {
+ List<ClusterNode> nodes = assignment.get(p);
+
+ if (nodes.size() > 0)
+ res.addAll(nodes);
+ }
+
+ nodes = res;
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Set<ClusterNode> primaryPartitionNodes() {
+ Set<ClusterNode> res = primaryPartsNodes;
- primaryPartsNodes0 = new HashSet<>();
+ if (res == null) {
+ res = new HashSet<>();
- for (int p = 0; p < parts; p++) {
+ for (int p = 0; p < assignment.size(); p++) {
List<ClusterNode> nodes = assignment.get(p);
if (nodes.size() > 0)
- primaryPartsNodes0.add(nodes.get(0));
+ res.add(nodes.get(0));
}
- primaryPartsNodes = primaryPartsNodes0;
+ primaryPartsNodes = res;
}
- return primaryPartsNodes0;
+ return res;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3bbc984/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
index e502dd5..94eaab4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/HistoryAffinityAssignment.java
@@ -30,6 +30,7 @@ import java.util.UUID;
/**
*
*/
+@SuppressWarnings("ForLoopReplaceableByForEach")
public class HistoryAffinityAssignment implements AffinityAssignment {
/** */
private final AffinityTopologyVersion topVer;
@@ -97,6 +98,20 @@ public class HistoryAffinityAssignment implements AffinityAssignment {
}
/** {@inheritDoc} */
+ @Override public Set<ClusterNode> nodes() {
+ Set<ClusterNode> res = new HashSet<>();
+
+ for (int p = 0; p < assignment.size(); p++) {
+ List<ClusterNode> nodes = assignment.get(p);
+
+ if (!F.isEmpty(nodes))
+ res.addAll(nodes);
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
@Override public Set<ClusterNode> primaryPartitionNodes() {
Set<ClusterNode> res = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3bbc984/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 8e994aa..29e9b27 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -400,7 +400,7 @@ public class GridReduceQueryExecutor {
// Explicit partitions mapping is not applicable to replicated cache.
if (cctx.isReplicated()) {
- for (ClusterNode clusterNode : cctx.affinity().assignment(topVer).primaryPartitionNodes())
+ for (ClusterNode clusterNode : cctx.affinity().assignment(topVer).nodes())
mapping.put(clusterNode, null);
return mapping;
@@ -478,35 +478,27 @@ public class GridReduceQueryExecutor {
if (F.isEmpty(extraNodes))
throw new CacheException("Failed to find data nodes for cache: " + extraCacheName);
- if (isReplicatedOnly && extraCctx.isReplicated()) {
- nodes.retainAll(extraNodes);
+ boolean disjoint;
- if (map.isEmpty()) {
- if (isPreloadingActive(cacheIds))
- return null; // Retry.
- else
- throw new CacheException("Caches have distinct sets of data nodes [cache1=" + cctx.name() +
- ", cache2=" + extraCacheName + "]");
+ if (extraCctx.isReplicated()) {
+ if (isReplicatedOnly) {
+ nodes.retainAll(extraNodes);
+
+ disjoint = map.isEmpty();
}
- }
- else if (!isReplicatedOnly && extraCctx.isReplicated()) {
- if (!extraNodes.containsAll(nodes))
- if (isPreloadingActive(cacheIds))
- return null; // Retry.
- else
- throw new CacheException("Caches have distinct sets of data nodes [cache1=" + cctx.name() +
- ", cache2=" + extraCacheName + "]");
- }
- else if (!isReplicatedOnly && !extraCctx.isReplicated()) {
- if (!extraNodes.equals(nodes))
- if (isPreloadingActive(cacheIds))
- return null; // Retry.
- else
- throw new CacheException("Caches have distinct sets of data nodes [cache1=" + cctx.name() +
- ", cache2=" + extraCacheName + "]");
+ else
+ disjoint = !extraNodes.containsAll(nodes);
}
else
- throw new IllegalStateException();
+ disjoint = !extraNodes.equals(nodes);
+
+ if (disjoint) {
+ if (isPreloadingActive(cacheIds))
+ return null; // Retry.
+ else
+ throw new CacheException("Caches have distinct sets of data nodes [cache1=" + cctx.name() +
+ ", cache2=" + extraCacheName + "]");
+ }
}
return map;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3bbc984/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest.java
new file mode 100644
index 0000000..a5b3706
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.replicated;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ * Tests non-collocated join with REPLICATED cache and no primary partitions for that cache on some nodes.
+ */
+@SuppressWarnings("unused")
+public class IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest extends GridCommonAbstractTest {
+ /** Client node name. */
+ public static final String NODE_CLI = "client";
+
+ /** */
+ public static final String CACHE_PARTITIONED = "partitioned";
+
+ /** */
+ public static final String CACHE_REPLICATED = "replicated";
+
+ /** */
+ public static final int REP_CNT = 3;
+
+ /** */
+ public static final int PART_CNT = 100;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setClientMode(F.eq(NODE_CLI, igniteInstanceName));
+
+ CacheConfiguration<Integer, PartValue> ccfg1 = new CacheConfiguration<>(CACHE_PARTITIONED);
+
+ ccfg1.setCacheMode(PARTITIONED);
+ ccfg1.setAtomicityMode(TRANSACTIONAL);
+ ccfg1.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg1.setIndexedTypes(Integer.class, PartValue.class);
+ ccfg1.setSqlSchema(QueryUtils.DFLT_SCHEMA);
+
+ CacheConfiguration<Integer, RepValue> ccfg2 = new CacheConfiguration<>(CACHE_REPLICATED);
+
+ ccfg2.setAffinity(new RendezvousAffinityFunction(false, REP_CNT));
+ ccfg2.setCacheMode(REPLICATED);
+ ccfg2.setAtomicityMode(TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(FULL_SYNC);
+ ccfg2.setIndexedTypes(Integer.class, RepValue.class);
+ ccfg2.setSqlSchema(QueryUtils.DFLT_SCHEMA);
+
+ cfg.setCacheConfiguration(ccfg1, ccfg2);
+
+ return cfg;
+ }
+
+ /**{@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ startGridsMultiThreaded(3);
+
+ Ignite cli = startGrid(NODE_CLI);
+
+ for (int i = 0; i < REP_CNT; i++)
+ cli.cache(CACHE_REPLICATED).put(i, new RepValue(i));
+
+ for (int i = 0; i < PART_CNT; i++)
+ cli.cache(CACHE_PARTITIONED).put(i, new PartValue(i, ((i + 1) % REP_CNT)));
+ }
+
+ /**{@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * Test non-colocated join.
+ *
+ * @throws Exception If failed.
+ */
+ public void testJoinNonCollocated() throws Exception {
+ SqlFieldsQuery qry = new SqlFieldsQuery("SELECT COUNT(*) FROM PartValue p, RepValue r WHERE p.repId=r.id");
+
+ long cnt = (Long)grid(NODE_CLI).cache(CACHE_PARTITIONED).query(qry).getAll().get(0).get(0);
+
+ assertEquals(PART_CNT, cnt);
+ }
+
+ /**
+ * Value for PARTITIONED cache.
+ */
+ public static class PartValue {
+ /** Id. */
+ @QuerySqlField
+ private int id;
+
+ /** Rep id. */
+ @QuerySqlField
+ private int repId;
+
+ /**
+ * @param id Id.
+ * @param repId Rep id.
+ */
+ public PartValue(int id, int repId) {
+ this.id = id;
+ this.repId = repId;
+ }
+ }
+
+ /**
+ * Value for REPLICATED cache.
+ */
+ public static class RepValue {
+ /** Id. */
+ @QuerySqlField
+ private int id;
+
+ /**
+ * @param id Id.
+ */
+ public RepValue(int id) {
+ this.id = id;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3bbc984/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 4d8016b..cb6426d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -90,6 +90,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheP
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCachePartitionedSnapshotEnabledQuerySelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryAbstractDistributedJoinSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNoRebalanceSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQueryP2PEnabledSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQueryROSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedFieldsQuerySelfTest;
@@ -299,6 +300,7 @@ public class IgniteCacheQuerySelfTestSuite extends TestSuite {
suite.addTestSuite(IgniteCacheReplicatedFieldsQuerySelfTest.class);
suite.addTestSuite(IgniteCacheReplicatedFieldsQueryROSelfTest.class);
suite.addTestSuite(IgniteCacheReplicatedFieldsQueryP2PEnabledSelfTest.class);
+ suite.addTestSuite(IgniteCacheReplicatedFieldsQueryJoinNoPrimaryPartitionsSelfTest.class);
suite.addTestSuite(IgniteCachePartitionedFieldsQuerySelfTest.class);
suite.addTestSuite(IgniteCacheAtomicFieldsQuerySelfTest.class);
suite.addTestSuite(IgniteCacheAtomicNearEnabledFieldsQuerySelfTest.class);
[28/50] [abbrv] ignite git commit: Minor: moved inner
TcpCommunicationSpi messages to top-level classes.
Posted by sb...@apache.org.
Minor: moved inner TcpCommunicationSpi messages to top-level classes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e358ae24
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e358ae24
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e358ae24
Branch: refs/heads/ignite-zk
Commit: e358ae241b334b6c03fc62ab0eee99f08eeb778a
Parents: c576d5a
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 13:50:19 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 13:50:19 2017 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 8 +-
.../spi/communication/tcp/HandshakeMessage.java | 156 ------------------
.../communication/tcp/HandshakeMessage2.java | 95 -----------
.../spi/communication/tcp/NodeIdMessage.java | 115 --------------
.../tcp/RecoveryLastReceivedMessage.java | 113 -------------
.../communication/tcp/TcpCommunicationSpi.java | 31 ++--
.../tcp/messages/HandshakeMessage.java | 157 +++++++++++++++++++
.../tcp/messages/HandshakeMessage2.java | 95 +++++++++++
.../tcp/messages/NodeIdMessage.java | 128 +++++++++++++++
.../messages/RecoveryLastReceivedMessage.java | 114 ++++++++++++++
.../tcp/messages/package-info.java | 22 +++
11 files changed, 541 insertions(+), 493 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 791dd91..78cb7a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -153,11 +153,11 @@ import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageFactory;
import org.apache.ignite.spi.collision.jobstealing.JobStealingRequest;
-import org.apache.ignite.spi.communication.tcp.HandshakeMessage;
-import org.apache.ignite.spi.communication.tcp.HandshakeMessage2;
-import org.apache.ignite.spi.communication.tcp.NodeIdMessage;
-import org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage;
+import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
+import org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage;
+import org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage;
import org.jsr166.ConcurrentHashMap8;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java
deleted file mode 100644
index 00e8e46..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.communication.tcp;
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-
-/**
- * Handshake message.
- */
-public class HandshakeMessage implements Message {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Message body size in bytes. */
- private static final int MESSAGE_SIZE = 32;
-
- /** Full message size (with message type) in bytes. */
- public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
-
- /** */
- private UUID nodeId;
-
- /** */
- private long rcvCnt;
-
- /** */
- private long connectCnt;
-
- /**
- * Default constructor required by {@link Message}.
- */
- public HandshakeMessage() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- * @param connectCnt Connect count.
- * @param rcvCnt Number of received messages.
- */
- public HandshakeMessage(UUID nodeId, long connectCnt, long rcvCnt) {
- assert nodeId != null;
- assert rcvCnt >= 0 : rcvCnt;
-
- this.nodeId = nodeId;
- this.connectCnt = connectCnt;
- this.rcvCnt = rcvCnt;
- }
-
- /**
- * @return Connection index.
- */
- public int connectionIndex() {
- return 0;
- }
-
- /**
- * @return Connect count.
- */
- public long connectCount() {
- return connectCnt;
- }
-
- /**
- * @return Number of received messages.
- */
- public long received() {
- return rcvCnt;
- }
-
- /**
- * @return Node ID.
- */
- public UUID nodeId() {
- return nodeId;
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- if (buf.remaining() < MESSAGE_FULL_SIZE)
- return false;
-
- TcpCommunicationSpi.writeMessageType(buf, directType());
-
- byte[] bytes = U.uuidToBytes(nodeId);
-
- assert bytes.length == 16 : bytes.length;
-
- buf.put(bytes);
-
- buf.putLong(rcvCnt);
-
- buf.putLong(connectCnt);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (buf.remaining() < MESSAGE_SIZE)
- return false;
-
- byte[] nodeIdBytes = new byte[NodeIdMessage.MESSAGE_SIZE];
-
- buf.get(nodeIdBytes);
-
- nodeId = U.bytesToUuid(nodeIdBytes, 0);
-
- rcvCnt = buf.getLong();
-
- connectCnt = buf.getLong();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return TcpCommunicationSpi.HANDSHAKE_MSG_TYPE;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HandshakeMessage.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java
deleted file mode 100644
index 1e8fdd9..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/HandshakeMessage2.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.communication.tcp;
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-
-/**
- * Updated handshake message.
- */
-public class HandshakeMessage2 extends HandshakeMessage {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private int connIdx;
-
- /**
- *
- */
- public HandshakeMessage2() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- * @param connectCnt Connect count.
- * @param rcvCnt Number of received messages.
- * @param connIdx Connection index.
- */
- HandshakeMessage2(UUID nodeId, long connectCnt, long rcvCnt, int connIdx) {
- super(nodeId, connectCnt, rcvCnt);
-
- this.connIdx = connIdx;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return -44;
- }
-
- /** {@inheritDoc} */
- @Override public int connectionIndex() {
- return connIdx;
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- if (!super.writeTo(buf, writer))
- return false;
-
- if (buf.remaining() < 4)
- return false;
-
- buf.putInt(connIdx);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (!super.readFrom(buf, reader))
- return false;
-
- if (buf.remaining() < 4)
- return false;
-
- connIdx = buf.getInt();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(HandshakeMessage2.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java
deleted file mode 100644
index d05b7ff..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/NodeIdMessage.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.communication.tcp;
-
-import java.nio.ByteBuffer;
-import java.util.UUID;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-
-/**
- * Node ID message.
- */
-public class NodeIdMessage implements Message {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** Message body size (with message type) in bytes. */
- static final int MESSAGE_SIZE = 16;
-
- /** Full message size (with message type) in bytes. */
- public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
-
- /** */
- byte[] nodeIdBytes;
-
- /** */
- byte[] nodeIdBytesWithType;
-
- /** */
- public NodeIdMessage() {
- // No-op.
- }
-
- /**
- * @param nodeId Node ID.
- */
- NodeIdMessage(UUID nodeId) {
- assert nodeId != null;
-
- nodeIdBytes = U.uuidToBytes(nodeId);
-
- assert nodeIdBytes.length == MESSAGE_SIZE : "Node ID size must be " + MESSAGE_SIZE;
-
- nodeIdBytesWithType = new byte[MESSAGE_FULL_SIZE];
-
- nodeIdBytesWithType[0] = (byte)(TcpCommunicationSpi.NODE_ID_MSG_TYPE & 0xFF);
- nodeIdBytesWithType[1] = (byte)((TcpCommunicationSpi.NODE_ID_MSG_TYPE >> 8) & 0xFF);
-
- System.arraycopy(nodeIdBytes, 0, nodeIdBytesWithType, 2, nodeIdBytes.length);
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- assert nodeIdBytes.length == MESSAGE_SIZE;
-
- if (buf.remaining() < MESSAGE_FULL_SIZE)
- return false;
-
- TcpCommunicationSpi.writeMessageType(buf, directType());
-
- buf.put(nodeIdBytes);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (buf.remaining() < MESSAGE_SIZE)
- return false;
-
- nodeIdBytes = new byte[MESSAGE_SIZE];
-
- buf.get(nodeIdBytes);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return TcpCommunicationSpi.NODE_ID_MSG_TYPE;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(NodeIdMessage.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java
deleted file mode 100644
index 5460084..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/RecoveryLastReceivedMessage.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.communication.tcp;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.plugin.extensions.communication.MessageReader;
-import org.apache.ignite.plugin.extensions.communication.MessageWriter;
-
-/**
- * Recovery acknowledgment message.
- */
-public class RecoveryLastReceivedMessage implements Message {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- static final long ALREADY_CONNECTED = -1;
-
- /** */
- static final long NODE_STOPPING = -2;
-
- /** Need wait. */
- static final long NEED_WAIT = -3;
-
- /** Message body size in bytes. */
- private static final int MESSAGE_SIZE = 8;
-
- /** Full message size (with message type) in bytes. */
- public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
-
- /** */
- private long rcvCnt;
-
- /**
- * Default constructor required by {@link Message}.
- */
- public RecoveryLastReceivedMessage() {
- // No-op.
- }
-
- /**
- * @param rcvCnt Number of received messages.
- */
- public RecoveryLastReceivedMessage(long rcvCnt) {
- this.rcvCnt = rcvCnt;
- }
-
- /**
- * @return Number of received messages.
- */
- public long received() {
- return rcvCnt;
- }
-
- /** {@inheritDoc} */
- @Override public void onAckReceived() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
- if (buf.remaining() < MESSAGE_FULL_SIZE)
- return false;
-
- TcpCommunicationSpi.writeMessageType(buf, directType());
-
- buf.putLong(rcvCnt);
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
- if (buf.remaining() < MESSAGE_SIZE)
- return false;
-
- rcvCnt = buf.getLong();
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public short directType() {
- return TcpCommunicationSpi.RECOVERY_LAST_ID_MSG_TYPE;
- }
-
- /** {@inheritDoc} */
- @Override public byte fieldsCount() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(RecoveryLastReceivedMessage.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 1f0061f..69da9ca 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -132,6 +132,10 @@ import org.apache.ignite.spi.IgniteSpiThread;
import org.apache.ignite.spi.IgniteSpiTimeoutObject;
import org.apache.ignite.spi.communication.CommunicationListener;
import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage;
+import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
+import org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage;
+import org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage;
import org.apache.ignite.spi.discovery.DiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.thread.IgniteThread;
@@ -141,9 +145,9 @@ import org.jsr166.ConcurrentLinkedDeque8;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
-import static org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
-import static org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage.NEED_WAIT;
-import static org.apache.ignite.spi.communication.tcp.RecoveryLastReceivedMessage.NODE_STOPPING;
+import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
+import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NEED_WAIT;
+import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NODE_STOPPING;
/**
* <tt>TcpCommunicationSpi</tt> is default communication SPI which uses
@@ -456,7 +460,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
ConnectionKey connKey;
if (msg instanceof NodeIdMessage) {
- sndId = U.bytesToUuid(((NodeIdMessage) msg).nodeIdBytes, 0);
+ sndId = U.bytesToUuid(((NodeIdMessage)msg).nodeIdBytes(), 0);
connKey = new ConnectionKey(sndId, 0, -1);
}
else {
@@ -3550,10 +3554,10 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
if (isSslEnabled()) {
assert sslHnd != null;
- ch.write(sslHnd.encrypt(ByteBuffer.wrap(nodeIdMessage().nodeIdBytesWithType)));
+ ch.write(sslHnd.encrypt(ByteBuffer.wrap(NodeIdMessage.nodeIdBytesWithType(safeLocalNodeId()))));
}
else
- ch.write(ByteBuffer.wrap(nodeIdMessage().nodeIdBytesWithType));
+ ch.write(ByteBuffer.wrap(NodeIdMessage.nodeIdBytesWithType(safeLocalNodeId())));
}
if (recovery != null) {
@@ -3806,20 +3810,27 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @return Node ID message.
*/
private NodeIdMessage nodeIdMessage() {
+ return new NodeIdMessage(safeLocalNodeId());
+ }
+
+ /**
+ * @return Local node ID.
+ */
+ private UUID safeLocalNodeId() {
ClusterNode locNode = getLocalNode();
UUID id;
if (locNode == null) {
U.warn(log, "Local node is not started or fully initialized [isStopping=" +
- getSpiContext().isStopping() + ']');
+ getSpiContext().isStopping() + ']');
id = new UUID(0, 0);
}
else
id = locNode.id();
- return new NodeIdMessage(id);
+ return id;
}
/** {@inheritDoc} */
@@ -3923,7 +3934,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
* @param buf Byte buffer.
* @param type Message type.
*/
- static void writeMessageType(ByteBuffer buf, short type) {
+ public static void writeMessageType(ByteBuffer buf, short type) {
buf.put((byte)(type & 0xFF));
buf.put((byte)((type >> 8) & 0xFF));
}
@@ -4426,7 +4437,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
out.write(U.IGNITE_HEADER);
writeMessageType(out, NODE_ID_MSG_TYPE);
- out.write(msg.nodeIdBytes);
+ out.write(msg.nodeIdBytes());
out.flush();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java
new file mode 100644
index 0000000..e3be9c9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.messages;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+/**
+ * Handshake message.
+ */
+public class HandshakeMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Message body size in bytes. */
+ private static final int MESSAGE_SIZE = 32;
+
+ /** Full message size (with message type) in bytes. */
+ public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
+
+ /** */
+ private UUID nodeId;
+
+ /** */
+ private long rcvCnt;
+
+ /** */
+ private long connectCnt;
+
+ /**
+ * Default constructor required by {@link Message}.
+ */
+ public HandshakeMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param connectCnt Connect count.
+ * @param rcvCnt Number of received messages.
+ */
+ public HandshakeMessage(UUID nodeId, long connectCnt, long rcvCnt) {
+ assert nodeId != null;
+ assert rcvCnt >= 0 : rcvCnt;
+
+ this.nodeId = nodeId;
+ this.connectCnt = connectCnt;
+ this.rcvCnt = rcvCnt;
+ }
+
+ /**
+ * @return Connection index.
+ */
+ public int connectionIndex() {
+ return 0;
+ }
+
+ /**
+ * @return Connect count.
+ */
+ public long connectCount() {
+ return connectCnt;
+ }
+
+ /**
+ * @return Number of received messages.
+ */
+ public long received() {
+ return rcvCnt;
+ }
+
+ /**
+ * @return Node ID.
+ */
+ public UUID nodeId() {
+ return nodeId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ if (buf.remaining() < MESSAGE_FULL_SIZE)
+ return false;
+
+ TcpCommunicationSpi.writeMessageType(buf, directType());
+
+ byte[] bytes = U.uuidToBytes(nodeId);
+
+ assert bytes.length == 16 : bytes.length;
+
+ buf.put(bytes);
+
+ buf.putLong(rcvCnt);
+
+ buf.putLong(connectCnt);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (buf.remaining() < MESSAGE_SIZE)
+ return false;
+
+ byte[] nodeIdBytes = new byte[NodeIdMessage.MESSAGE_SIZE];
+
+ buf.get(nodeIdBytes);
+
+ nodeId = U.bytesToUuid(nodeIdBytes, 0);
+
+ rcvCnt = buf.getLong();
+
+ connectCnt = buf.getLong();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TcpCommunicationSpi.HANDSHAKE_MSG_TYPE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HandshakeMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java
new file mode 100644
index 0000000..f27a825
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.messages;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Updated handshake message.
+ */
+public class HandshakeMessage2 extends HandshakeMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private int connIdx;
+
+ /**
+ *
+ */
+ public HandshakeMessage2() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param connectCnt Connect count.
+ * @param rcvCnt Number of received messages.
+ * @param connIdx Connection index.
+ */
+ public HandshakeMessage2(UUID nodeId, long connectCnt, long rcvCnt, int connIdx) {
+ super(nodeId, connectCnt, rcvCnt);
+
+ this.connIdx = connIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return -44;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int connectionIndex() {
+ return connIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (buf.remaining() < 4)
+ return false;
+
+ buf.putInt(connIdx);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ if (buf.remaining() < 4)
+ return false;
+
+ connIdx = buf.getInt();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(HandshakeMessage2.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java
new file mode 100644
index 0000000..2c6aa30
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.messages;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+/**
+ * Node ID message.
+ */
+public class NodeIdMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Message body size (with message type) in bytes. */
+ static final int MESSAGE_SIZE = 16;
+
+ /** Full message size (with message type) in bytes. */
+ public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
+
+ /** */
+ private byte[] nodeIdBytes;
+
+ /** */
+ public NodeIdMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param nodeId Node ID.
+ */
+ public NodeIdMessage(UUID nodeId) {
+ assert nodeId != null;
+
+ nodeIdBytes = U.uuidToBytes(nodeId);
+
+ assert nodeIdBytes.length == MESSAGE_SIZE : "Node ID size must be " + MESSAGE_SIZE;
+ }
+
+ /**
+ * @return Node ID bytes.
+ */
+ public byte[] nodeIdBytes() {
+ return nodeIdBytes;
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @return Marshalled node ID bytes with direct message type.
+ */
+ public static byte[] nodeIdBytesWithType(UUID nodeId) {
+ byte[] nodeIdBytesWithType = new byte[MESSAGE_FULL_SIZE];
+
+ nodeIdBytesWithType[0] = (byte)(TcpCommunicationSpi.NODE_ID_MSG_TYPE & 0xFF);
+ nodeIdBytesWithType[1] = (byte)((TcpCommunicationSpi.NODE_ID_MSG_TYPE >> 8) & 0xFF);
+
+ U.uuidToBytes(nodeId, nodeIdBytesWithType, 2);
+
+ return nodeIdBytesWithType;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ assert nodeIdBytes.length == MESSAGE_SIZE;
+
+ if (buf.remaining() < MESSAGE_FULL_SIZE)
+ return false;
+
+ TcpCommunicationSpi.writeMessageType(buf, directType());
+
+ buf.put(nodeIdBytes);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (buf.remaining() < MESSAGE_SIZE)
+ return false;
+
+ nodeIdBytes = new byte[MESSAGE_SIZE];
+
+ buf.get(nodeIdBytes);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TcpCommunicationSpi.NODE_ID_MSG_TYPE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(NodeIdMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java
new file mode 100644
index 0000000..eef2655
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.messages;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+
+/**
+ * Recovery acknowledgment message.
+ */
+public class RecoveryLastReceivedMessage implements Message {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ public static final long ALREADY_CONNECTED = -1;
+
+ /** */
+ public static final long NODE_STOPPING = -2;
+
+ /** Need wait. */
+ public static final long NEED_WAIT = -3;
+
+ /** Message body size in bytes. */
+ private static final int MESSAGE_SIZE = 8;
+
+ /** Full message size (with message type) in bytes. */
+ public static final int MESSAGE_FULL_SIZE = MESSAGE_SIZE + DIRECT_TYPE_SIZE;
+
+ /** */
+ private long rcvCnt;
+
+ /**
+ * Default constructor required by {@link Message}.
+ */
+ public RecoveryLastReceivedMessage() {
+ // No-op.
+ }
+
+ /**
+ * @param rcvCnt Number of received messages.
+ */
+ public RecoveryLastReceivedMessage(long rcvCnt) {
+ this.rcvCnt = rcvCnt;
+ }
+
+ /**
+ * @return Number of received messages.
+ */
+ public long received() {
+ return rcvCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onAckReceived() {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ if (buf.remaining() < MESSAGE_FULL_SIZE)
+ return false;
+
+ TcpCommunicationSpi.writeMessageType(buf, directType());
+
+ buf.putLong(rcvCnt);
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ if (buf.remaining() < MESSAGE_SIZE)
+ return false;
+
+ rcvCnt = buf.getLong();
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return TcpCommunicationSpi.RECOVERY_LAST_ID_MSG_TYPE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(RecoveryLastReceivedMessage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e358ae24/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/package-info.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/package-info.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/package-info.java
new file mode 100644
index 0000000..662dd26
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains implementation messages.
+ */
+package org.apache.ignite.spi.communication.tcp.messages;
\ No newline at end of file
[27/50] [abbrv] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-zk-ce
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-zk-ce
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0b225226
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0b225226
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0b225226
Branch: refs/heads/ignite-zk
Commit: 0b225226dc5d14b0d6afcb5a13dfc3ca71442a42
Parents: 2b74163 c576d5a
Author: sboikov <sb...@gridgain.com>
Authored: Fri Dec 15 11:29:58 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Dec 15 11:29:58 2017 +0300
----------------------------------------------------------------------
.../communication/GridIoMessageFactory.java | 14 +-
.../spi/communication/tcp/HandshakeMessage.java | 156 ++++++++
.../communication/tcp/HandshakeMessage2.java | 95 +++++
.../spi/communication/tcp/NodeIdMessage.java | 115 ++++++
.../tcp/RecoveryLastReceivedMessage.java | 113 ++++++
.../communication/tcp/TcpCommunicationSpi.java | 391 +------------------
6 files changed, 492 insertions(+), 392 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b225226/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0b225226/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
[36/50] [abbrv] ignite git commit: GridNioServer: ses can be null on
close if async connect failed
Posted by sb...@apache.org.
GridNioServer: ses can be null on close if async connect failed
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/edc66af8
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/edc66af8
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/edc66af8
Branch: refs/heads/ignite-zk
Commit: edc66af8221508e8d892bd9bd1e550ada9ac4a47
Parents: 91be7af
Author: sboikov <sb...@gridgain.com>
Authored: Mon Dec 18 14:41:46 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Mon Dec 18 14:41:46 2017 +0300
----------------------------------------------------------------------
.../ignite/internal/util/nio/GridNioServer.java | 59 +++++++++++---------
1 file changed, 34 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/edc66af8/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 1d595d2..97f6020 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -2293,7 +2293,11 @@ public class GridNioServer<T> {
else if (log.isDebugEnabled())
log.debug("Failed to process selector key [ses=" + ses + ", err=" + e + ']');
- close(ses, new GridNioException(e));
+ // Can be null if async connect failed.
+ if (ses != null)
+ close(ses, new GridNioException(e));
+ else
+ closeKey(key);
}
}
}
@@ -2517,6 +2521,34 @@ public class GridNioServer<T> {
}
/**
+ * @param key Key.
+ */
+ private void closeKey(SelectionKey key) {
+ // Shutdown input and output so that remote client will see correct socket close.
+ Socket sock = ((SocketChannel)key.channel()).socket();
+
+ try {
+ try {
+ sock.shutdownInput();
+ }
+ catch (IOException ignored) {
+ // No-op.
+ }
+
+ try {
+ sock.shutdownOutput();
+ }
+ catch (IOException ignored) {
+ // No-op.
+ }
+ }
+ finally {
+ U.close(key, log);
+ U.close(sock, log);
+ }
+ }
+
+ /**
* Closes the session and all associated resources, then notifies the listener.
*
* @param ses Session to be closed.
@@ -2536,8 +2568,6 @@ public class GridNioServer<T> {
sessions.remove(ses);
workerSessions.remove(ses);
- SelectionKey key = ses.key();
-
if (ses.setClosed()) {
ses.onClosed();
@@ -2549,28 +2579,7 @@ public class GridNioServer<T> {
((DirectBuffer)ses.readBuffer()).cleaner().clean();
}
- // Shutdown input and output so that remote client will see correct socket close.
- Socket sock = ((SocketChannel)key.channel()).socket();
-
- try {
- try {
- sock.shutdownInput();
- }
- catch (IOException ignored) {
- // No-op.
- }
-
- try {
- sock.shutdownOutput();
- }
- catch (IOException ignored) {
- // No-op.
- }
- }
- finally {
- U.close(key, log);
- U.close(sock, log);
- }
+ closeKey(ses.key());
if (e != null)
filterChain.onExceptionCaught(ses, e);
[07/50] [abbrv] ignite git commit: IGNITE-6423: PDS could be
corrupted if partition have been evicted and owned again. This closes #3115.
Posted by sb...@apache.org.
IGNITE-6423: PDS could be corrupted if partition have been evicted and owned again. This closes #3115.
Fixed page memory update operations without checkpoint lock.
Fixed page CRC calculation.
Fixed outdated page handling.
Added checkpoint lock hold assertions for memory update operations.
Fixed incorrect tests.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e24d4d03
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e24d4d03
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e24d4d03
Branch: refs/heads/ignite-zk
Commit: e24d4d03dd00cc464e6e793e2a3f23988664a0cd
Parents: 24412f5
Author: Andrey V. Mashenkov <an...@gmail.com>
Authored: Wed Dec 13 14:16:50 2017 +0300
Committer: Andrey V. Mashenkov <an...@gmail.com>
Committed: Wed Dec 13 14:16:50 2017 +0300
----------------------------------------------------------------------
.../internal/pagemem/store/PageStore.java | 5 +-
.../processors/cache/GridCacheTtlManager.java | 3 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 41 +++--
.../distributed/dht/GridDhtTxPrepareFuture.java | 5 +
.../local/atomic/GridLocalAtomicCache.java | 171 ++++++++++---------
.../GridCacheDatabaseSharedManager.java | 9 +-
.../persistence/GridCacheOffheapManager.java | 4 +
.../processors/cache/persistence/RowStore.java | 2 +
.../cache/persistence/file/FilePageStore.java | 32 +++-
.../persistence/file/FilePageStoreManager.java | 7 +-
.../persistence/pagemem/PageMemoryImpl.java | 56 ++++--
.../processors/cache/tree/CacheDataTree.java | 2 +
.../cache/tree/PendingEntriesTree.java | 2 +
.../pagemem/impl/PageMemoryNoLoadSelfTest.java | 10 +-
.../cache/persistence/DummyPageIO.java | 41 +++++
.../IgnitePdsContinuousRestartTest.java | 5 -
...gnitePdsRecoveryAfterFileCorruptionTest.java | 83 ++++++---
...ckpointSimulationWithRealCpDisabledTest.java | 53 +++++-
.../db/file/IgnitePdsEvictionTest.java | 30 ++++
.../query/h2/database/H2TreeIndex.java | 58 ++++---
20 files changed, 439 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
index f6e577c..42d584d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/PageStore.java
@@ -71,10 +71,11 @@ public interface PageStore {
* @param pageId Page ID.
* @param pageBuf Page buffer to write.
* @param tag Partition file version, 1-based incrementing counter. For outdated pages {@code tag} has lower value,
- * and write does nothing
+ * and write does nothing.
+ * @param calculateCrc if {@code False} crc calculation will be forcibly skipped.
* @throws IgniteCheckedException If page writing failed (IO error occurred).
*/
- public void write(long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException;
+ public void write(long pageId, ByteBuffer pageBuf, int tag, boolean calculateCrc) throws IgniteCheckedException;
/**
* Gets page offset within the store file.
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
index b006154..9c013fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
@@ -23,7 +23,6 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.util.GridConcurrentSkipListSet;
import org.apache.ignite.internal.util.lang.IgniteInClosure2X;
import org.apache.ignite.internal.util.typedef.X;
@@ -145,7 +144,7 @@ public class GridCacheTtlManager extends GridCacheManagerAdapter {
try {
X.println(">>>");
X.println(">>> TTL processor memory stats [igniteInstanceName=" + cctx.igniteInstanceName() +
- ", cache=" + cctx.name() + ']');
+ ", cache=" + cctx.name() + ']');
X.println(">>> pendingEntriesSize: " + pendingSize());
}
catch (IgniteCheckedException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 370a92e..8ad6d4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -139,20 +139,20 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
- public void onCacheStarted(GridCacheContext cctx) throws IgniteCheckedException{
+ public void onCacheStarted(GridCacheContext cctx) throws IgniteCheckedException {
if (cctx.affinityNode() && cctx.ttl().eagerTtlEnabled() && pendingEntries == null) {
String name = "PendingEntries";
- long rootPage = allocateForTree();
+ long rootPage = allocateForTree();
- pendingEntries = new PendingEntriesTree(
- grp,
- name,
- grp.dataRegion().pageMemory(),
- rootPage,
- grp.reuseList(),
- true);
- }
+ pendingEntries = new PendingEntriesTree(
+ grp,
+ name,
+ grp.dataRegion().pageMemory(),
+ rootPage,
+ grp.reuseList(),
+ true);
+ }
}
/**
@@ -196,6 +196,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
try {
if (grp.sharedGroup()) {
assert cacheId != CU.UNDEFINED_CACHE_ID;
+ assert ctx.database().checkpointLockIsHeldByThread();
for (CacheDataStore store : cacheDataStores())
store.clear(cacheId);
@@ -437,7 +438,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
while (it.hasNext()) {
cctx.shared().database().checkpointReadLock();
- try{
+ try {
KeyCacheObject key = it.next().key();
try {
@@ -1192,6 +1193,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
try {
int cacheId = grp.sharedGroup() ? cctx.cacheId() : CU.UNDEFINED_CACHE_ID;
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
dataTree.invoke(new SearchRow(cacheId, key), CacheDataRowAdapter.RowData.NO_KEY, c);
switch (c.operationType()) {
@@ -1232,8 +1235,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
CacheObject val,
GridCacheVersion ver,
long expireTime,
- @Nullable CacheDataRow oldRow) throws IgniteCheckedException
- {
+ @Nullable CacheDataRow oldRow) throws IgniteCheckedException {
int cacheId = grp.storeCacheIdInDataPage() ? cctx.cacheId() : CU.UNDEFINED_CACHE_ID;
DataRow dataRow = new DataRow(key, val, ver, partId, expireTime, cacheId);
@@ -1258,8 +1260,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
}
/** {@inheritDoc} */
- @Override public void update(GridCacheContext cctx,KeyCacheObject key,
-
+ @Override public void update(
+ GridCacheContext cctx,
+ KeyCacheObject key,
CacheObject val,
GridCacheVersion ver,
long expireTime,
@@ -1285,6 +1288,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
CacheDataRow old;
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
if (canUpdateOldRow(cctx, oldRow, dataRow) && rowStore.updateRow(oldRow.link(), dataRow)) {
old = oldRow;
@@ -1363,6 +1368,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
try {
int cacheId = grp.sharedGroup() ? cctx.cacheId() : CU.UNDEFINED_CACHE_ID;
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
CacheDataRow oldRow = dataTree.remove(new SearchRow(cacheId, key));
finishRemove(cctx, key, oldRow);
@@ -1425,8 +1432,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
return dataTree.find(null, null);
}
- /** {@inheritDoc}
- * @param cacheId*/
+ /** {@inheritDoc} */
@Override public GridCursor<? extends CacheDataRow> cursor(int cacheId) throws IgniteCheckedException {
return cursor(cacheId, null, null);
}
@@ -1486,6 +1492,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
/** {@inheritDoc} */
@Override public void clear(int cacheId) throws IgniteCheckedException {
assert cacheId != CU.UNDEFINED_CACHE_ID;
+ assert ctx.database().checkpointLockIsHeldByThread();
if (cacheSize(cacheId) == 0)
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index 6873890..0fb9ee4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1819,6 +1819,8 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
GridDrType drType = cacheCtx.isDrEnabled() ? GridDrType.DR_PRELOAD : GridDrType.DR_NONE;
+ cctx.database().checkpointReadLock();
+
try {
if (entry.initialValue(info.value(),
info.version(),
@@ -1850,6 +1852,9 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
log.debug("Failed to set entry initial value (entry is obsolete, " +
"will retry): " + entry);
}
+ finally {
+ cctx.database().checkpointReadUnlock();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index 599a58c..1454e96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -830,113 +830,120 @@ public class GridLocalAtomicCache<K, V> extends GridLocalCache<K, V> {
CacheEntryPredicate[] filters = CU.filterArray(filter);
- ctx.shared().database().ensureFreeSpace(ctx.dataRegion());
-
- if (writeThrough && keys.size() > 1) {
- return updateWithBatch(op,
- keys,
- vals,
- invokeArgs,
- expiryPlc,
- ver,
- filters,
- keepBinary,
- subjId,
- taskName);
- }
-
- Iterator<?> valsIter = vals != null ? vals.iterator() : null;
-
IgniteBiTuple<Boolean, ?> res = null;
CachePartialUpdateCheckedException err = null;
- boolean intercept = ctx.config().getInterceptor() != null;
+ ctx.shared().database().checkpointReadLock();
- for (K key : keys) {
- if (key == null)
- throw new NullPointerException("Null key.");
+ try {
+ ctx.shared().database().ensureFreeSpace(ctx.dataRegion());
+
+ if (writeThrough && keys.size() > 1) {
+ return updateWithBatch(op,
+ keys,
+ vals,
+ invokeArgs,
+ expiryPlc,
+ ver,
+ filters,
+ keepBinary,
+ subjId,
+ taskName);
+ }
- Object val = valsIter != null ? valsIter.next() : null;
+ Iterator<?> valsIter = vals != null ? vals.iterator() : null;
- if (val == null && op != DELETE)
- throw new NullPointerException("Null value.");
+ boolean intercept = ctx.config().getInterceptor() != null;
- KeyCacheObject cacheKey = ctx.toCacheKeyObject(key);
+ for (K key : keys) {
+ if (key == null)
+ throw new NullPointerException("Null key.");
- if (op == UPDATE) {
- val = ctx.toCacheObject(val);
+ Object val = valsIter != null ? valsIter.next() : null;
- ctx.validateKeyAndValue(cacheKey, (CacheObject)val);
- }
- else if (op == TRANSFORM)
- ctx.kernalContext().resource().inject(val, GridResourceIoc.AnnotationSet.ENTRY_PROCESSOR, ctx.name());
+ if (val == null && op != DELETE)
+ throw new NullPointerException("Null value.");
- while (true) {
- GridCacheEntryEx entry = null;
+ KeyCacheObject cacheKey = ctx.toCacheKeyObject(key);
- try {
- entry = entryEx(cacheKey);
-
- GridTuple3<Boolean, Object, EntryProcessorResult<Object>> t = entry.innerUpdateLocal(
- ver,
- val == null ? DELETE : op,
- val,
- invokeArgs,
- writeThrough,
- readThrough,
- retval,
- keepBinary,
- expiryPlc,
- true,
- true,
- filters,
- intercept,
- subjId,
- taskName);
+ if (op == UPDATE) {
+ val = ctx.toCacheObject(val);
- if (op == TRANSFORM) {
- if (t.get3() != null) {
- Map<K, EntryProcessorResult> computedMap;
+ ctx.validateKeyAndValue(cacheKey, (CacheObject)val);
+ }
+ else if (op == TRANSFORM)
+ ctx.kernalContext().resource().inject(val, GridResourceIoc.AnnotationSet.ENTRY_PROCESSOR, ctx.name());
- if (res == null) {
- computedMap = U.newHashMap(keys.size());
+ while (true) {
+ GridCacheEntryEx entry = null;
- res = new IgniteBiTuple<>(true, computedMap);
- }
- else
- computedMap = (Map<K, EntryProcessorResult>)res.get2();
+ try {
+ entry = entryEx(cacheKey);
- computedMap.put(key, t.get3());
+ GridTuple3<Boolean, Object, EntryProcessorResult<Object>> t = entry.innerUpdateLocal(
+ ver,
+ val == null ? DELETE : op,
+ val,
+ invokeArgs,
+ writeThrough,
+ readThrough,
+ retval,
+ keepBinary,
+ expiryPlc,
+ true,
+ true,
+ filters,
+ intercept,
+ subjId,
+ taskName);
+
+ if (op == TRANSFORM) {
+ if (t.get3() != null) {
+ Map<K, EntryProcessorResult> computedMap;
+
+ if (res == null) {
+ computedMap = U.newHashMap(keys.size());
+
+ res = new IgniteBiTuple<>(true, computedMap);
+ }
+ else
+ computedMap = (Map<K, EntryProcessorResult>)res.get2();
+
+ computedMap.put(key, t.get3());
+ }
}
- }
- else if (res == null)
- res = new T2(t.get1(), t.get2());
+ else if (res == null)
+ res = new T2(t.get1(), t.get2());
- break; // While.
- }
- catch (GridCacheEntryRemovedException ignored) {
- if (log.isDebugEnabled())
- log.debug("Got removed entry while updating (will retry): " + key);
+ break; // While.
+ }
+ catch (GridCacheEntryRemovedException ignored) {
+ if (log.isDebugEnabled())
+ log.debug("Got removed entry while updating (will retry): " + key);
- entry = null;
- }
- catch (IgniteCheckedException e) {
- if (err == null)
- err = partialUpdateException();
+ entry = null;
+ }
+ catch (IgniteCheckedException e) {
+ if (err == null)
+ err = partialUpdateException();
- err.add(F.asList(key), e);
+ err.add(F.asList(key), e);
- U.error(log, "Failed to update key : " + key, e);
+ U.error(log, "Failed to update key : " + key, e);
- break;
- }
- finally {
- if (entry != null)
- ctx.evicts().touch(entry, ctx.affinity().affinityTopologyVersion());
+ break;
+ }
+ finally {
+ if (entry != null)
+ ctx.evicts().touch(entry, ctx.affinity().affinityTopologyVersion());
+ }
}
}
}
+ finally {
+ ctx.shared().database().checkpointReadUnlock();
+ }
if (err != null)
throw err;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index c0e59bc..3c2842f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1850,7 +1850,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (tag != null) {
tmpWriteBuf.rewind();
- PageStore store = storeMgr.writeInternal(fullId.groupId(), fullId.pageId(), tmpWriteBuf, tag);
+ PageStore store = storeMgr.writeInternal(fullId.groupId(), fullId.pageId(), tmpWriteBuf, tag, true);
tmpWriteBuf.rewind();
@@ -2640,6 +2640,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
if (tag != null) {
+ assert PageIO.getType(tmpWriteBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
+ assert PageIO.getVersion(tmpWriteBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+
tmpWriteBuf.rewind();
if (persStoreMetrics.metricsEnabled()) {
@@ -2661,9 +2664,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
tmpWriteBuf.rewind();
- PageIO.setCrc(writeAddr, 0);
-
- PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag);
+ PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, false);
updStores.add(store);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index cfa1829..e818b00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -922,6 +922,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
reuseRoot.pageId().pageId(),
reuseRoot.isAllocated()) {
@Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+ assert grp.shared().database().checkpointLockIsHeldByThread();
+
return pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA);
}
};
@@ -938,6 +940,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
treeRoot.pageId().pageId(),
treeRoot.isAllocated()) {
@Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+ assert grp.shared().database().checkpointLockIsHeldByThread();
+
return pageMem.allocatePage(grpId, partId, PageIdAllocator.FLAG_DATA);
}
};
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
index 2051021..ad2f731 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
@@ -107,6 +107,8 @@ public class RowStore {
* @return {@code True} if was able to update row.
*/
public boolean updateRow(long link, CacheDataRow row) throws IgniteCheckedException {
+ assert !persistenceEnabled || ctx.database().checkpointLockIsHeldByThread();
+
return freeList.updateDataRow(link, row);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 408240c..47f1d4d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -433,7 +433,7 @@ public class FilePageStore implements PageStore {
}
/** {@inheritDoc} */
- @Override public void write(long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
+ @Override public void write(long pageId, ByteBuffer pageBuf, int tag, boolean calculateCrc) throws IgniteCheckedException {
init();
lock.readLock().lock();
@@ -450,13 +450,20 @@ public class FilePageStore implements PageStore {
assert pageBuf.capacity() == pageSize;
assert pageBuf.position() == 0;
assert pageBuf.order() == ByteOrder.nativeOrder();
- assert PageIO.getCrc(pageBuf) == 0 : U.hexLong(pageId);
+ assert PageIO.getType(pageBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(pageId);
+ assert PageIO.getVersion(pageBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(pageId);
- int crc32 = skipCrc ? 0 : PureJavaCrc32.calcCrc32(pageBuf, pageSize);
+ if (calculateCrc && !skipCrc) {
+ assert PageIO.getCrc(pageBuf) == 0 : U.hexLong(pageId);
- PageIO.setCrc(pageBuf, crc32);
+ PageIO.setCrc(pageBuf, calcCrc32(pageBuf, pageSize));
+ }
- pageBuf.position(0);
+ // Check whether crc was calculated somewhere above the stack if it is forcibly skipped.
+ assert skipCrc || PageIO.getCrc(pageBuf) != 0 || calcCrc32(pageBuf, pageSize) == 0 :
+ "CRC hasn't been calculated, crc=0";
+
+ assert pageBuf.position() == 0 : pageBuf.position();
int len = pageSize;
@@ -480,6 +487,21 @@ public class FilePageStore implements PageStore {
}
}
+ /**
+ * @param pageBuf Page buffer.
+ * @param pageSize Page size.
+ */
+ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) {
+ try {
+ pageBuf.position(0);
+
+ return PureJavaCrc32.calcCrc32(pageBuf, pageSize);
+ }
+ finally {
+ pageBuf.position(0);
+ }
+ }
+
/** {@inheritDoc} */
@Override public long pageOffset(long pageId) {
return (long) PageIdUtils.pageIndex(pageId) * pageSize + headerSize();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 1fe22ca..66af0dd 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -291,7 +291,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/** {@inheritDoc} */
@Override public void write(int grpId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
- writeInternal(grpId, pageId, pageBuf, tag);
+ writeInternal(grpId, pageId, pageBuf, tag, true);
}
/** {@inheritDoc} */
@@ -306,15 +306,16 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
* @param pageId Page ID.
* @param pageBuf Page buffer.
* @param tag Partition tag (growing 1-based partition file version). Used to validate page is not outdated
+ * @param calculateCrc if {@code False} crc calculation will be forcibly skipped.
* @return PageStore to which the page has been written.
* @throws IgniteCheckedException If IO error occurred.
*/
- public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
+ public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int tag, boolean calculateCrc) throws IgniteCheckedException {
int partId = PageIdUtils.partId(pageId);
PageStore store = getStore(cacheId, partId);
- store.write(pageId, pageBuf, tag);
+ store.write(pageId, pageBuf, tag, calculateCrc);
return store;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 3014099..41de7f0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -173,7 +173,7 @@ public class PageMemoryImpl implements PageMemoryEx {
private final int sysPageSize;
/** Shared context. */
- private final GridCacheSharedContext<?, ?> sharedCtx;
+ private final GridCacheSharedContext<?, ?> ctx;
/** State checker. */
private final CheckpointLockStateChecker stateChecker;
@@ -236,7 +236,7 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param directMemoryProvider Memory allocator to use.
- * @param sharedCtx Cache shared context.
+ * @param ctx Cache shared context.
* @param pageSize Page size.
* @param flushDirtyPage Callback invoked when a dirty page is evicted.
* @param changeTracker Callback invoked to track changes in pages.
@@ -245,7 +245,7 @@ public class PageMemoryImpl implements PageMemoryEx {
public PageMemoryImpl(
DirectMemoryProvider directMemoryProvider,
long[] sizes,
- GridCacheSharedContext<?, ?> sharedCtx,
+ GridCacheSharedContext<?, ?> ctx,
int pageSize,
GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage,
GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
@@ -253,11 +253,11 @@ public class PageMemoryImpl implements PageMemoryEx {
DataRegionMetricsImpl memMetrics,
boolean throttleEnabled
) {
- assert sharedCtx != null;
+ assert ctx != null;
- log = sharedCtx.logger(PageMemoryImpl.class);
+ log = ctx.logger(PageMemoryImpl.class);
- this.sharedCtx = sharedCtx;
+ this.ctx = ctx;
this.directMemoryProvider = directMemoryProvider;
this.sizes = sizes;
this.flushDirtyPage = flushDirtyPage;
@@ -265,8 +265,8 @@ public class PageMemoryImpl implements PageMemoryEx {
this.stateChecker = stateChecker;
this.throttleEnabled = throttleEnabled;
- storeMgr = sharedCtx.pageStore();
- walMgr = sharedCtx.wal();
+ storeMgr = ctx.pageStore();
+ walMgr = ctx.wal();
assert storeMgr != null;
assert walMgr != null;
@@ -336,15 +336,15 @@ public class PageMemoryImpl implements PageMemoryEx {
*
*/
private void initWriteThrottle() {
- if (!(sharedCtx.database() instanceof GridCacheDatabaseSharedManager)) {
+ if (!(ctx.database() instanceof GridCacheDatabaseSharedManager)) {
log.error("Write throttle can't start. Unexpected class of database manager: " +
- sharedCtx.database().getClass());
+ ctx.database().getClass());
throttleEnabled = false;
}
if (throttleEnabled)
- writeThrottle = new PagesWriteThrottle(this, (GridCacheDatabaseSharedManager)sharedCtx.database());
+ writeThrottle = new PagesWriteThrottle(this, (GridCacheDatabaseSharedManager)ctx.database());
}
/** {@inheritDoc} */
@@ -423,6 +423,8 @@ public class PageMemoryImpl implements PageMemoryEx {
flags == PageIdAllocator.FLAG_IDX && partId == PageIdAllocator.INDEX_PARTITION :
"flags = " + flags + ", partId = " + partId;
+ assert ctx.database().checkpointLockIsHeldByThread();
+
long pageId = storeMgr.allocatePage(cacheId, partId, flags);
memMetrics.incrementTotalAllocatedPages();
@@ -441,7 +443,19 @@ public class PageMemoryImpl implements PageMemoryEx {
boolean isTrackingPage = trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
try {
- long relPtr = seg.borrowOrAllocateFreePage(pageId);
+ long relPtr = seg.loadedPages.get(
+ cacheId,
+ PageIdUtils.effectivePageId(pageId),
+ seg.partTag(cacheId, partId),
+ INVALID_REL_PTR,
+ OUTDATED_REL_PTR
+ );
+
+ if (relPtr == OUTDATED_REL_PTR)
+ relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
+
+ if (relPtr == INVALID_REL_PTR)
+ relPtr = seg.borrowOrAllocateFreePage(pageId);
if (relPtr == INVALID_REL_PTR)
relPtr = seg.evictPage();
@@ -470,8 +484,8 @@ public class PageMemoryImpl implements PageMemoryEx {
if (PageIO.getType(pageAddr) == 0) {
trackingIO.initNewPage(pageAddr, pageId, pageSize());
- if (!sharedCtx.wal().isAlwaysWriteFullPages())
- sharedCtx.wal().log(
+ if (!ctx.wal().isAlwaysWriteFullPages())
+ ctx.wal().log(
new InitNewPageRecord(
cacheId,
pageId,
@@ -480,7 +494,7 @@ public class PageMemoryImpl implements PageMemoryEx {
)
);
else
- sharedCtx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize()));
+ ctx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize()));
}
}
@@ -991,6 +1005,9 @@ public class PageMemoryImpl implements PageMemoryEx {
PageHeader.releasePage(absPtr);
}
+ assert PageIO.getType(tmpBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
+ assert PageIO.getVersion(tmpBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+
return true;
}
finally {
@@ -1229,6 +1246,9 @@ public class PageMemoryImpl implements PageMemoryEx {
pageSize()
);
+ assert PageIO.getType(tmpAbsPtr + PAGE_OVERHEAD) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
+ assert PageIO.getVersion(tmpAbsPtr + PAGE_OVERHEAD) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+
PageHeader.dirty(absPtr, false);
PageHeader.tempBufferPointer(absPtr, tmpRelPtr);
@@ -1269,6 +1289,10 @@ public class PageMemoryImpl implements PageMemoryEx {
long pageId = PageIO.getPageId(page + PAGE_OVERHEAD);
+ assert pageId != 0 : U.hexLong(PageHeader.readPageId(page));
+ assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
+ assert PageIO.getType(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId);
+
try {
rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
@@ -1390,6 +1414,8 @@ public class PageMemoryImpl implements PageMemoryEx {
boolean wasDirty = PageHeader.dirty(absPtr, dirty);
if (dirty) {
+ assert ctx.database().checkpointLockIsHeldByThread();
+
if (!wasDirty || forceAdd) {
boolean added = segment(pageId.groupId(), pageId.pageId()).dirtyPages.add(pageId);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
index afa3fd7..f2bfa41 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
@@ -77,6 +77,8 @@ public class CacheDataTree extends BPlusTree<CacheSearchRow, CacheDataRow> {
this.rowStore = rowStore;
this.grp = grp;
+ assert !grp.dataRegion().config().isPersistenceEnabled() || grp.shared().database().checkpointLockIsHeldByThread();
+
initTree(initNew);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
index a6ec6e7..0b1c931 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
@@ -64,6 +64,8 @@ public class PendingEntriesTree extends BPlusTree<PendingRow, PendingRow> {
this.grp = grp;
+ assert !grp.dataRegion().config().isPersistenceEnabled() || grp.shared().database().checkpointLockIsHeldByThread();
+
initTree(initNew);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
index 3b9e393..0f3bf9c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
@@ -33,6 +33,7 @@ import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -47,6 +48,9 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest {
/** */
private static final int MAX_MEMORY_SIZE = 10 * 1024 * 1024;
+ /** */
+ private static final PageIO PAGE_IO = new DummyPageIO();
+
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false));
@@ -226,6 +230,8 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest {
assertNotNull(pageAddr);
try {
+ PAGE_IO.initNewPage(pageAddr, id.pageId(), mem.pageSize());
+
long updId = PageIdUtils.rotatePageId(id.pageId());
PageIO.setPageId(pageAddr, updId);
@@ -334,7 +340,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest {
long pageAddr = mem.writeLock(-1, pageId, page);
try {
- PageIO.setPageId(pageAddr, pageId);
+ PAGE_IO.initNewPage(pageAddr, pageId, mem.pageSize());
for (int i = PageIO.COMMON_HEADER_END; i < PAGE_SIZE; i++)
PageUtils.putByte(pageAddr, i, (byte)val);
@@ -355,7 +361,7 @@ public class PageMemoryNoLoadSelfTest extends GridCommonAbstractTest {
long pageAddr = mem.readLock(-1, pageId, page);
- assert(pageAddr != 0);
+ assert pageAddr != 0;
try {
for (int i = PageIO.COMMON_HEADER_END; i < PAGE_SIZE; i++) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/DummyPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/DummyPageIO.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/DummyPageIO.java
new file mode 100644
index 0000000..1b36ac1
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/DummyPageIO.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.util.GridStringBuilder;
+
+/**
+ * Dummy PageIO implementation. For test purposes only.
+ */
+public class DummyPageIO extends PageIO {
+ /** */
+ public DummyPageIO() {
+ super(2 * Short.MAX_VALUE, 1);
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ protected void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
+ sb.a("DummyPageIO [\n");
+ sb.a("addr=").a(addr).a(", ");
+ sb.a("pageSize=").a(addr);
+ sb.a("\n]");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
index 27b1950..fa89bf2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
@@ -221,11 +221,6 @@ public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
checkRebalancingDuringLoad(10, 500, 8, 16);
}
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return TimeUnit.MINUTES.toMillis(3);
- }
-
/**
* @throws Exception if failed.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
index 9369443..8e20585 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -142,20 +142,33 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
PageMemory mem = sharedCtx.database().dataRegion(policyName).pageMemory();
+ DummyPageIO pageIO = new DummyPageIO();
+
int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
FullPageId[] pages = new FullPageId[totalPages];
- for (int i = 0; i < totalPages; i++)
- pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+ // Get lock to prevent assertion. A new page should be allocated under checkpoint lock.
+ psMgr.checkpointReadLock();
- generateWal(
- (PageMemoryImpl)mem,
- sharedCtx.pageStore(),
- sharedCtx.wal(),
- cacheId,
- pages
- );
+ try {
+ for (int i = 0; i < totalPages; i++) {
+ pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+
+ initPage(mem, pageIO, pages[i]);
+ }
+
+ generateWal(
+ (PageMemoryImpl)mem,
+ sharedCtx.pageStore(),
+ sharedCtx.wal(),
+ cacheId,
+ pages
+ );
+ }
+ finally {
+ psMgr.checkpointReadUnlock();
+ }
eraseDataFromDisk(pageStore, cacheId, pages[0]);
@@ -169,6 +182,31 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
}
/**
+ * Initializes page.
+ * @param mem page memory implementation.
+ * @param pageIO page io implementation.
+ * @param fullId full page id.
+ * @throws IgniteCheckedException if error occurs.
+ */
+ private void initPage(PageMemory mem, PageIO pageIO, FullPageId fullId) throws IgniteCheckedException {
+ long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
+
+ try {
+ final long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
+
+ try {
+ pageIO.initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
+ }
+ finally {
+ mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.groupId(), fullId.pageId(), page);
+ }
+ }
+
+ /**
* @param pageStore Page store.
* @param cacheId Cache id.
* @param page Page.
@@ -207,21 +245,28 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
PageMemory mem = shared.database().dataRegion(null).pageMemory();
- for (FullPageId fullId : pages) {
- long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
+ dbMgr.checkpointReadLock();
- try {
- long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page);
+ try {
+ for (FullPageId fullId : pages) {
+ long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
+
+ try {
+ long pageAddr = mem.readLock(fullId.groupId(), fullId.pageId(), page);
- for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
- assertEquals(j + (int)fullId.pageId(), PageUtils.getInt(pageAddr, j));
+ for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
+ assertEquals(j + (int)fullId.pageId(), PageUtils.getInt(pageAddr, j));
- mem.readUnlock(fullId.groupId(), fullId.pageId(), page);
- }
- finally {
- mem.releasePage(fullId.groupId(), fullId.pageId(), page);
+ mem.readUnlock(fullId.groupId(), fullId.pageId(), page);
+ }
+ finally {
+ mem.releasePage(fullId.groupId(), fullId.pageId(), page);
+ }
}
}
+ finally {
+ dbMgr.checkpointReadUnlock();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
index 5ae8969..0dd9153 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
@@ -47,7 +47,9 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
@@ -519,10 +521,12 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
PageMemoryEx mem = (PageMemoryEx) dbMgr.dataRegion(null).pageMemory();
- ig.context().cache().context().database().checkpointReadLock();
-
FullPageId[] pageIds = new FullPageId[100];
+ DummyPageIO pageIO = new DummyPageIO();
+
+ ig.context().cache().context().database().checkpointReadLock();
+
try {
for (int i = 0; i < pageIds.length; i++)
pageIds[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
@@ -535,9 +539,9 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
- PageIO.setPageId(pageAddr, fullId.pageId());
-
try {
+ pageIO.initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
+
assertTrue(mem.isDirty(fullId.groupId(), fullId.pageId(), page));
}
finally {
@@ -737,8 +741,22 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
Set<FullPageId> allocated = new HashSet<>();
+ IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+ PageIO pageIO = new DummyPageIO();
+
for (int i = 0; i < TOTAL_PAGES; i++) {
- FullPageId fullId = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+ FullPageId fullId;
+
+ db.checkpointReadLock();
+ try {
+ fullId = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+
+ initPage(mem, pageIO, fullId);
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
resMap.put(fullId, -1);
@@ -982,4 +1000,29 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
private void deleteWorkFiles() throws IgniteCheckedException {
deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
}
+
+ /**
+ * Initializes page.
+ * @param mem page memory implementation.
+ * @param pageIO page io implementation.
+ * @param fullId full page id.
+ * @throws IgniteCheckedException if error occurs.
+ */
+ private void initPage(PageMemory mem, PageIO pageIO, FullPageId fullId) throws IgniteCheckedException {
+ long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
+
+ try {
+ final long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
+
+ try {
+ pageIO.initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
+ }
+ finally {
+ mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.groupId(), fullId.pageId(), page);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
index 47a4b7b..1b86e3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.DummyPageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -138,6 +139,8 @@ public class IgnitePdsEvictionTest extends GridCommonAbstractTest {
IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
+ PageIO pageIO = new DummyPageIO();
+
// Allocate.
for (int i = 0; i < size; i++) {
db.checkpointReadLock();
@@ -145,6 +148,8 @@ public class IgnitePdsEvictionTest extends GridCommonAbstractTest {
final FullPageId fullId = new FullPageId(memory.allocatePage(cacheId, i % 256, PageMemory.FLAG_DATA),
cacheId);
+ initPage(memory, pageIO, fullId);
+
pageIds.add(fullId);
}
finally {
@@ -180,6 +185,31 @@ public class IgnitePdsEvictionTest extends GridCommonAbstractTest {
}
/**
+ * Initializes page.
+ * @param mem page memory implementation.
+ * @param pageIO page io implementation.
+ * @param fullId full page id.
+ * @throws IgniteCheckedException if error occurs.
+ */
+ private void initPage(PageMemory mem, PageIO pageIO, FullPageId fullId) throws IgniteCheckedException {
+ long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
+
+ try {
+ final long pageAddr = mem.writeLock(fullId.groupId(), fullId.pageId(), page);
+
+ try {
+ pageIO.initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
+ }
+ finally {
+ mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.groupId(), fullId.pageId(), page);
+ }
+ }
+
+ /**
* @param start Start index.
* @param end End index.
* @param memory PageMemory.
http://git-wip-us.apache.org/repos/asf/ignite/blob/e24d4d03/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
index edfaecf..53d5de5 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
@@ -27,6 +27,7 @@ import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.RootPage;
import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
@@ -109,26 +110,35 @@ public class H2TreeIndex extends GridH2IndexBase {
segments = new H2Tree[segmentsCnt];
+ IgniteCacheDatabaseSharedManager db = cctx.shared().database();
+
for (int i = 0; i < segments.length; i++) {
- RootPage page = getMetaPage(name, i);
-
- segments[i] = new H2Tree(
- name,
- cctx.offheap().reuseListForIndex(name),
- cctx.groupId(),
- cctx.dataRegion().pageMemory(),
- cctx.shared().wal(),
- cctx.offheap().globalRemoveId(),
- tbl.rowFactory(),
- page.pageId().pageId(),
- page.isAllocated(),
- cols,
- inlineIdxs,
- computeInlineSize(inlineIdxs, inlineSize)) {
- @Override public int compareValues(Value v1, Value v2) {
- return v1 == v2 ? 0 : table.compareTypeSafe(v1, v2);
- }
- };
+ db.checkpointReadLock();
+
+ try {
+ RootPage page = getMetaPage(name, i);
+
+ segments[i] = new H2Tree(
+ name,
+ cctx.offheap().reuseListForIndex(name),
+ cctx.groupId(),
+ cctx.dataRegion().pageMemory(),
+ cctx.shared().wal(),
+ cctx.offheap().globalRemoveId(),
+ tbl.rowFactory(),
+ page.pageId().pageId(),
+ page.isAllocated(),
+ cols,
+ inlineIdxs,
+ computeInlineSize(inlineIdxs, inlineSize)) {
+ @Override public int compareValues(Value v1, Value v2) {
+ return v1 == v2 ? 0 : table.compareTypeSafe(v1, v2);
+ }
+ };
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
}
}
else {
@@ -202,6 +212,8 @@ public class H2TreeIndex extends GridH2IndexBase {
H2Tree tree = treeForRead(seg);
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
return tree.put(row);
}
catch (IgniteCheckedException e) {
@@ -221,6 +233,8 @@ public class H2TreeIndex extends GridH2IndexBase {
H2Tree tree = treeForRead(seg);
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
return tree.putx(row);
}
catch (IgniteCheckedException e) {
@@ -240,6 +254,8 @@ public class H2TreeIndex extends GridH2IndexBase {
H2Tree tree = treeForRead(seg);
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
return tree.remove(row);
}
catch (IgniteCheckedException e) {
@@ -259,6 +275,8 @@ public class H2TreeIndex extends GridH2IndexBase {
H2Tree tree = treeForRead(seg);
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
return tree.removex(row);
}
catch (IgniteCheckedException e) {
@@ -326,6 +344,8 @@ public class H2TreeIndex extends GridH2IndexBase {
@Override public void destroy(boolean rmvIndex) {
try {
if (cctx.affinityNode() && rmvIndex) {
+ assert cctx.shared().database().checkpointLockIsHeldByThread();
+
for (int i = 0; i < segments.length; i++) {
H2Tree tree = segments[i];
[31/50] [abbrv] ignite git commit: IGNITE-7212 Fixed infinite loop in
TCP communication SPI skipping local address
Posted by sb...@apache.org.
IGNITE-7212 Fixed infinite loop in TCP communication SPI skipping local address
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/86dc36d0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/86dc36d0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/86dc36d0
Branch: refs/heads/ignite-zk
Commit: 86dc36d036f664ace223d3e97445b0016ccab3c0
Parents: e358ae2
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Dec 15 16:37:50 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Dec 15 16:37:50 2017 +0300
----------------------------------------------------------------------
.../apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/86dc36d0/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 69da9ca..3c5b5e9 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3107,7 +3107,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
log.debug("Skipping local address [addr=" + addr +
", locAddrs=" + node.attribute(createSpiAttributeName(ATTR_ADDRS)) +
", node=" + node + ']');
- continue;
+ break;
}
boolean needWait = false;
[50/50] [abbrv] ignite git commit: Throttle new events generation
Posted by sb...@apache.org.
Throttle new events generation
(cherry picked from commit d29ea50)
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d6ec00c0
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d6ec00c0
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d6ec00c0
Branch: refs/heads/ignite-zk
Commit: d6ec00c0e46bcac71a9a01b749e481fa656ed708
Parents: f9698f4
Author: sboikov <sb...@gridgain.com>
Authored: Wed Dec 20 11:03:51 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Wed Dec 20 11:03:51 2017 +0300
----------------------------------------------------------------------
.../discovery/zk/internal/ZookeeperDiscoveryImpl.java | 2 +-
.../zk/internal/ZookeeperDiscoverySpiBasicTest.java | 14 ++++++++++++++
2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d6ec00c0/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index a72d742..c54d7c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -93,7 +93,7 @@ public class ZookeeperDiscoveryImpl {
static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD";
/** */
- private static final String IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS = "IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS";
+ static final String IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS = "IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS";
/** */
private static final String IGNITE_ZOOKEEPER_DISCOVERY_EVTS_THROTTLE = "IGNITE_ZOOKEEPER_DISCOVERY_EVTS_THROTTLE";
http://git-wip-us.apache.org/repos/asf/ignite/blob/d6ec00c0/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index 0630af4..1373da8 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -1006,6 +1006,20 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * @throws Exception If failed.
+ */
+ public void testConcurrentStartStop2_EventsThrottle() throws Exception {
+ System.setProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS, "1");
+
+ try {
+ concurrentStartStop(5);
+ }
+ finally {
+ System.clearProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_MAX_EVTS);
+ }
+ }
+
+ /**
* @param initNodes Number of initially started nnodes.
* @throws Exception If failed.
*/
[44/50] [abbrv] ignite git commit: IGNITE-7242: Broken javadoc for KNN
Posted by sb...@apache.org.
IGNITE-7242: Broken javadoc for KNN
this closes #3251
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7782d381
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7782d381
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7782d381
Branch: refs/heads/ignite-zk
Commit: 7782d3814bf3e3b8f94ff0c274912ce5b44e7be6
Parents: ec29023
Author: YuriBabak <y....@gmail.com>
Authored: Tue Dec 19 13:57:24 2017 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Tue Dec 19 13:57:24 2017 +0300
----------------------------------------------------------------------
.../ignite/ml/knn/regression/KNNMultipleLinearRegression.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7782d381/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
index 2db8a9f..90392a3 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNMultipleLinearRegression.java
@@ -36,7 +36,7 @@ import org.apache.ignite.ml.structures.LabeledVector;
* </ul>
*/
public class KNNMultipleLinearRegression extends KNNModel {
- /** {@inheritDoc} */
+ /** */
public KNNMultipleLinearRegression(int k, DistanceMeasure distanceMeasure, KNNStrategy stgy,
LabeledDataset training) {
super(k, distanceMeasure, stgy, training);
[11/50] [abbrv] ignite git commit: IGNITE-6495: performance
measurement of decision trees algorithms
Posted by sb...@apache.org.
IGNITE-6495: performance measurement of decision trees algorithms
this closes #3213
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a0516f9e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a0516f9e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a0516f9e
Branch: refs/heads/ignite-zk
Commit: a0516f9e8ccb39125de67e5196b943ea6d55f7df
Parents: 47e7daf
Author: Oleg Ignatenko <oi...@gridgain.com>
Authored: Wed Dec 13 20:10:29 2017 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Wed Dec 13 20:10:29 2017 +0300
----------------------------------------------------------------------
.../yardstick/config/benchmark-ml.properties | 2 +
modules/yardstick/pom-standalone.xml | 12 +
modules/yardstick/pom.xml | 12 +
.../IgniteColumnDecisionTreeGiniBenchmark.java | 70 +++
...niteColumnDecisionTreeVarianceBenchmark.java | 71 ++++
.../yardstick/ml/trees/SplitDataGenerator.java | 426 +++++++++++++++++++
.../ignite/yardstick/ml/trees/package-info.java | 22 +
7 files changed, 615 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/config/benchmark-ml.properties
----------------------------------------------------------------------
diff --git a/modules/yardstick/config/benchmark-ml.properties b/modules/yardstick/config/benchmark-ml.properties
index dbf90dd..4867c0f 100644
--- a/modules/yardstick/config/benchmark-ml.properties
+++ b/modules/yardstick/config/benchmark-ml.properties
@@ -85,4 +85,6 @@ CONFIGS="\
-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteDenseLocalOffHeapMatrixMulBenchmark -sn IgniteNode -ds ${ver}dense-local-offheap-matrix-mul-${b}-backup,\
-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSparseLocalMatrixMulBenchmark -sn IgniteNode -ds ${ver}sparse-local-matrix-mul-${b}-backup,\
-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteSparseBlockDistributedMatrixMulBenchmark -sn IgniteNode -ds ${ver}sparse-block-distributed-matrix-mul-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteColumnDecisionTreeVarianceBenchmark -sn IgniteNode -ds ${ver}column-decision-tree-variance-${b}-backup,\
+-cfg ${SCRIPT_DIR}/../config/ignite-localhost-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteColumnDecisionTreeGiniBenchmark -sn IgniteNode -ds ${ver}column-decision-tree-gini-${b}-backup,\
"
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/pom-standalone.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom-standalone.xml b/modules/yardstick/pom-standalone.xml
index 78569a3..58920b9 100644
--- a/modules/yardstick/pom-standalone.xml
+++ b/modules/yardstick/pom-standalone.xml
@@ -123,6 +123,18 @@
<version>${project.version}</version>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/pom.xml
----------------------------------------------------------------------
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index 6a98584..8dda6a7 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -143,6 +143,18 @@
<version>${project.version}</version>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
new file mode 100644
index 0000000..f8a7c08
--- /dev/null
+++ b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.ml.trees;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
+import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+
+/**
+ * Ignite benchmark that performs ML Grid operations.
+ */
+@SuppressWarnings("unused")
+public class IgniteColumnDecisionTreeGiniBenchmark extends IgniteAbstractBenchmark {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
+ // because we create ignite cache internally.
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ this.getClass().getSimpleName(), new Runnable() {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixedGini
+ int totalPts = 1 << 10;
+ int featCnt = 2;
+
+ HashMap<Integer, Integer> catsInfo = new HashMap<>();
+ catsInfo.put(1, 3);
+
+ SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
+ featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
+ split(0, 1, new int[] {0, 2}).
+ split(1, 0, -10.0);
+
+ gen.testByGen(totalPts, ContinuousSplitCalculators.GINI.apply(ignite),
+ RegionCalculators.GINI, RegionCalculators.MEAN, ignite);
+ }
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
new file mode 100644
index 0000000..f9d417f
--- /dev/null
+++ b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.ml.trees;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
+import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
+
+/**
+ * Ignite benchmark that performs ML Grid operations.
+ */
+@SuppressWarnings("unused")
+public class IgniteColumnDecisionTreeVarianceBenchmark extends IgniteAbstractBenchmark {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public boolean test(Map<Object, Object> ctx) throws Exception {
+ // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
+ // because we create ignite cache internally.
+ IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+ this.getClass().getSimpleName(), new Runnable() {
+ /** {@inheritDoc} */
+ @Override public void run() {
+ // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixed
+ int totalPts = 1 << 10;
+ int featCnt = 2;
+
+ HashMap<Integer, Integer> catsInfo = new HashMap<>();
+ catsInfo.put(1, 3);
+
+ SplitDataGenerator<DenseLocalOnHeapVector> gen
+ = new SplitDataGenerator<>(
+ featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
+ split(0, 1, new int[] {0, 2}).
+ split(1, 0, -10.0);
+
+ gen.testByGen(totalPts,
+ ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, ignite);
+ }
+ });
+
+ igniteThread.start();
+
+ igniteThread.join();
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
new file mode 100644
index 0000000..f530300
--- /dev/null
+++ b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.yardstick.ml.trees;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.DoubleStream;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.math.StorageConstants;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.structures.LabeledVectorDouble;
+import org.apache.ignite.ml.trees.ContinuousRegionInfo;
+import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
+import org.apache.ignite.ml.trees.models.DecisionTreeModel;
+import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
+import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
+import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
+import org.apache.ignite.ml.util.Utils;
+
+/** */
+class SplitDataGenerator<V extends Vector> {
+ /** */
+ private static final Random rnd = new Random(12349L);
+
+ /** */
+ private static final double DELTA = 100.0;
+
+ /** Map of the form of (is categorical -> list of region indexes). */
+ private final Map<Boolean, List<Integer>> di;
+
+ /** List of regions. */
+ private final List<Region> regs;
+
+ /** Data of bounds of regions. */
+ private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
+
+ /** */
+ private final Map<Integer, Integer> catFeaturesInfo;
+
+ /** Supplier of vectors. */
+ private final Supplier<V> supplier;
+
+ /** Features count. */
+ private final int featCnt;
+
+ /**
+ * Create SplitDataGenerator.
+ *
+ * @param featCnt Features count.
+ * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
+ * count).
+ * @param supplier Supplier of vectors.
+ */
+ SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier) {
+ regs = new LinkedList<>();
+ boundsData = new HashMap<>();
+ this.supplier = supplier;
+ this.featCnt = featCnt;
+ this.catFeaturesInfo = catFeaturesInfo;
+
+ // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
+ di = IntStream.range(0, featCnt).
+ boxed().
+ collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
+
+ // Categorical coordinates info.
+ Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
+ di.get(true).forEach(i -> {
+ BitSet bs = new BitSet();
+ bs.set(0, catFeaturesInfo.get(i));
+ catCoords.put(i, new CatCoordInfo(bs));
+ });
+
+ // Continuous coordinates info.
+ Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
+ di.get(false).forEach(i -> {
+ contCoords.put(i, new ContCoordInfo());
+ boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
+ });
+
+ Region firstReg = new Region(catCoords, contCoords, 0);
+ regs.add(firstReg);
+ }
+
+ /** */
+ <D extends ContinuousRegionInfo> void testByGen(int totalPts,
+ IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
+ IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
+ IgniteFunction<DoubleStream, Double> regCalc, Ignite ignite) {
+
+ List<IgniteBiTuple<Integer, V>> lst = points(totalPts, (i, rn) -> i).collect(Collectors.toList());
+
+ Collections.shuffle(lst, rnd);
+
+ SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts,
+ featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
+
+ Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
+
+ int i = 0;
+ for (IgniteBiTuple<Integer, V> bt : lst) {
+ byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
+ byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
+ m.setRow(i, bt.get2().getStorage().data());
+ i++;
+ }
+
+ ColumnDecisionTreeTrainer<D> trainer =
+ new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
+
+ DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catFeaturesInfo));
+
+ byRegion.keySet().forEach(k -> mdl.predict(byRegion.get(k).get(0).features()));
+ }
+
+ /**
+ * Split region by continuous coordinate using given threshold.
+ *
+ * @param regIdx Region index.
+ * @param coordIdx Coordinate index.
+ * @param threshold Threshold.
+ * @return {@code this}.
+ */
+ SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
+ Region regToSplit = regs.get(regIdx);
+ ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
+
+ double left = cci.left;
+ double right = cci.right;
+
+ if (threshold < left || threshold > right)
+ throw new MathIllegalArgumentException("Threshold is out of region bounds.");
+
+ regToSplit.incTwoPow();
+
+ Region newReg = Utils.copy(regToSplit);
+ newReg.contCoords.get(coordIdx).left = threshold;
+
+ regs.add(regIdx + 1, newReg);
+ cci.right = threshold;
+
+ IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
+ double min = bounds.get1();
+ double max = bounds.get2();
+ boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
+
+ return this;
+ }
+
+ /**
+ * Split region by categorical coordinate.
+ *
+ * @param regIdx Region index.
+ * @param coordIdx Coordinate index.
+ * @param cats Categories allowed for the left sub region.
+ * @return {@code this}.
+ */
+ SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
+ BitSet subset = new BitSet();
+ Arrays.stream(cats).forEach(subset::set);
+ Region regToSplit = regs.get(regIdx);
+ CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
+
+ BitSet ssc = (BitSet)subset.clone();
+ BitSet set = cci.bs;
+ ssc.and(set);
+ if (ssc.length() != subset.length())
+ throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
+
+ ssc.xor(set);
+ set.and(subset);
+
+ regToSplit.incTwoPow();
+ Region newReg = Utils.copy(regToSplit);
+ newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
+
+ regs.add(regIdx + 1, newReg);
+
+ return this;
+ }
+
+ /**
+ * Get stream of points generated by this generator.
+ *
+ * @param ptsCnt Points count.
+ */
+ private Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
+ return IntStream.range(0, regs.size()).
+ boxed().
+ map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd)
+ .map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
+ }
+
+ /**
+ * Convert double array to {@link LabeledVectorDouble}
+ *
+ * @param arr Array for conversion.
+ * @return LabeledVectorDouble.
+ */
+ private static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
+ return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(
+ Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
+ }
+
+ /**
+ * Categorical coordinate info.
+ */
+ private static class CatCoordInfo implements Serializable {
+ /**
+ * Defines categories which are included in this region
+ */
+ private final BitSet bs;
+
+ /**
+ * Construct CatCoordInfo.
+ *
+ * @param bs Bitset.
+ */
+ CatCoordInfo(BitSet bs) {
+ this.bs = bs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "CatCoordInfo [" +
+ "bs=" + bs +
+ ']';
+ }
+ }
+
+ /**
+ * Continuous coordinate info.
+ */
+ private static class ContCoordInfo implements Serializable {
+ /**
+ * Left (min) bound of region.
+ */
+ private double left;
+
+ /**
+ * Right (max) bound of region.
+ */
+ private double right;
+
+ /**
+ * Construct ContCoordInfo.
+ */
+ ContCoordInfo() {
+ left = Double.NEGATIVE_INFINITY;
+ right = Double.POSITIVE_INFINITY;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ContCoordInfo [" +
+ "left=" + left +
+ ", right=" + right +
+ ']';
+ }
+ }
+
+ /**
+ * Class representing information about region.
+ */
+ private static class Region implements Serializable {
+ /**
+ * Information about categorical coordinates restrictions of this region in form of
+ * (coordinate index -> restriction)
+ */
+ private final Map<Integer, CatCoordInfo> catCoords;
+
+ /**
+ * Information about continuous coordinates restrictions of this region in form of
+ * (coordinate index -> restriction)
+ */
+ private final Map<Integer, ContCoordInfo> contCoords;
+
+ /**
+ * Region should contain {@code 1/2^twoPow * totalPoints} points.
+ */
+ private int twoPow;
+
+ /**
+ * Construct region by information about restrictions on coordinates (features) values.
+ *
+ * @param catCoords Restrictions on categorical coordinates.
+ * @param contCoords Restrictions on continuous coordinates
+ * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
+ */
+ Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
+ this.catCoords = catCoords;
+ this.contCoords = contCoords;
+ this.twoPow = twoPow;
+ }
+
+ /** */
+ int divideBy() {
+ return 1 << twoPow;
+ }
+
+ /** */
+ void incTwoPow() {
+ twoPow++;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "Region [" +
+ "catCoords=" + catCoords +
+ ", contCoords=" + contCoords +
+ ", twoPow=" + twoPow +
+ ']';
+ }
+
+ /**
+ * Generate continuous coordinate for this region.
+ *
+ * @param coordIdx Coordinate index.
+ * @param boundsData Data with bounds
+ * @param rnd Random numbers generator.
+ * @return Categorical coordinate value.
+ */
+ double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
+ Random rnd) {
+ ContCoordInfo cci = contCoords.get(coordIdx);
+ double left = cci.left;
+ double right = cci.right;
+
+ if (left == Double.NEGATIVE_INFINITY)
+ left = boundsData.get(coordIdx).get1() - DELTA;
+
+ if (right == Double.POSITIVE_INFINITY)
+ right = boundsData.get(coordIdx).get2() + DELTA;
+
+ double size = right - left;
+
+ return left + rnd.nextDouble() * size;
+ }
+
+ /**
+ * Generate categorical coordinate value for this region.
+ *
+ * @param coordIdx Coordinate index.
+ * @param rnd Random numbers generator.
+ * @return Categorical coordinate value.
+ */
+ double generateCatCoord(int coordIdx, Random rnd) {
+ // Pick random bit.
+ BitSet bs = catCoords.get(coordIdx).bs;
+ int j = rnd.nextInt(bs.length());
+
+ int i = 0;
+ int bn = 0;
+ int bnp = 0;
+
+ while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
+ i++;
+ bnp = bn;
+ bn++;
+ }
+
+ return bnp;
+ }
+
+ /**
+ * Generate points for this region.
+ *
+ * @param ptsCnt Count of points to generate.
+ * @param val Label for all points in this region.
+ * @param boundsData Data about bounds of continuous coordinates.
+ * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
+ * categories indexes).
+ * @param s Vectors supplier.
+ * @param rnd Random numbers generator.
+ * @param <V> Type of vectors.
+ * @return Stream of generated points for this region.
+ */
+ <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
+ Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
+ Supplier<V> s,
+ Random rnd) {
+ return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
+ V v = s.get();
+ int coordsCnt = v.size();
+ catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
+ catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
+
+ v.setX(coordsCnt - 1, val);
+ return v;
+ });
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/a0516f9e/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java
----------------------------------------------------------------------
diff --git a/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java
new file mode 100644
index 0000000..fc379a6
--- /dev/null
+++ b/modules/yardstick/src/main/ml/org/apache/ignite/yardstick/ml/trees/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * ML Grid decision tree benchmarks.
+ */
+package org.apache.ignite.yardstick.ml.trees;
\ No newline at end of file
[45/50] [abbrv] ignite git commit: zk
Posted by sb...@apache.org.
zk
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4525b921
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4525b921
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4525b921
Branch: refs/heads/ignite-zk
Commit: 4525b9218b17786b9ddf6a3932fc3422b8cfc735
Parents: 246b765
Author: sboikov <sb...@gridgain.com>
Authored: Tue Dec 19 13:34:22 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Tue Dec 19 14:34:25 2017 +0300
----------------------------------------------------------------------
.../DefaultCommunicationProblemResolver.java | 199 +++++++++++++++++++
.../configuration/IgniteConfiguration.java | 12 +-
.../apache/ignite/internal/IgniteKernal.java | 3 +
.../org/apache/ignite/internal/IgnitionEx.java | 3 +
.../internal/managers/GridManagerAdapter.java | 8 +
.../discovery/GridDiscoveryManager.java | 68 +++++++
.../managers/discovery/IgniteDiscoverySpi.java | 4 +-
.../org/apache/ignite/spi/IgniteSpiAdapter.java | 10 +
.../org/apache/ignite/spi/IgniteSpiContext.java | 4 +
.../communication/tcp/TcpCommunicationSpi.java | 18 +-
.../DefaultCommunicationProblemResolver.java | 174 ----------------
.../spi/discovery/tcp/TcpDiscoverySpi.java | 2 +-
.../spi/discovery/zk/ZookeeperDiscoverySpi.java | 4 +-
.../ZkCommunicationErrorProcessFuture.java | 2 +-
.../zk/internal/ZookeeperDiscoveryImpl.java | 33 +--
.../ZookeeperDiscoverySpiBasicTest.java | 102 ++++++++--
.../testframework/GridSpiTestContext.java | 10 +
17 files changed, 433 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
new file mode 100644
index 0000000..1e973d3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationProblemResolver.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.util.BitSet;
+import java.util.List;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ *
+ */
+public class DefaultCommunicationProblemResolver implements CommunicationProblemResolver {
+ /** {@inheritDoc} */
+ @Override public void resolve(CommunicationProblemContext ctx) {
+ ClusterGraph graph = new ClusterGraph(ctx);
+
+ BitSet cluster = graph.findLargestIndependentCluster();
+
+ List<ClusterNode> nodes = ctx.topologySnapshot();
+
+ if (graph.checkFullyConnected(cluster) && cluster.cardinality() < nodes.size()) {
+ for (int i = 0; i < nodes.size(); i++) {
+ if (!cluster.get(i))
+ ctx.killNode(nodes.get(i));
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class ClusterGraph {
+ /** */
+ private final static int WORD_IDX_SHIFT = 6;
+
+ /**
+ * @param bitIndex Bit index.
+ * @return Word index containing bit with given index.
+ */
+ private static int wordIndex(int bitIndex) {
+ return bitIndex >> WORD_IDX_SHIFT;
+ }
+
+ /** */
+ private final int nodeCnt;
+
+ /** */
+ private final long[] visitBitSet;
+
+ /** */
+ private final CommunicationProblemContext ctx;
+
+ /** */
+ private final List<ClusterNode> nodes;
+
+ /**
+ * @param ctx Context.
+ */
+ ClusterGraph(CommunicationProblemContext ctx) {
+ this.ctx = ctx;
+
+ nodes = ctx.topologySnapshot();
+
+ nodeCnt = nodes.size();
+
+ assert nodeCnt > 0;
+
+ visitBitSet = initBitSet(nodeCnt);
+ }
+
+ /**
+ * @param bitCnt Number of bits.
+ * @return Bit set words.
+ */
+ static long[] initBitSet(int bitCnt) {
+ return new long[wordIndex(bitCnt - 1) + 1];
+ }
+
+ /**
+ * @return Cluster nodes bit set.
+ */
+ BitSet findLargestIndependentCluster() {
+ BitSet maxCluster = null;
+ int maxClusterSize = 0;
+
+ for (int i = 0; i < nodeCnt; i++) {
+ if (getBit(visitBitSet, i))
+ continue;
+
+ BitSet cluster = new BitSet(nodeCnt);
+
+ search(cluster, i);
+
+ int size = cluster.cardinality();
+
+ if (maxCluster == null || size > maxClusterSize) {
+ maxCluster = cluster;
+ maxClusterSize = size;
+ }
+ }
+
+ return maxCluster;
+ }
+
+ /**
+ * @param cluster Cluster nodes bit set.
+ * @return {@code True} if all cluster nodes are able to connect to each other.
+ */
+ boolean checkFullyConnected(BitSet cluster) {
+ int startIdx = 0;
+
+ int clusterNodes = cluster.cardinality();
+
+ for (;;) {
+ int idx = cluster.nextSetBit(startIdx);
+
+ if (idx == -1)
+ break;
+
+ ClusterNode node1 = nodes.get(idx);
+
+ for (int i = 0; i < clusterNodes; i++) {
+ if (!cluster.get(i) || i == idx)
+ continue;
+
+ ClusterNode node2 = nodes.get(i);
+
+ if (cluster.get(i) && !ctx.connectionAvailable(node1, node2))
+ return false;
+ }
+
+ startIdx = idx + 1;
+ }
+
+ return true;
+ }
+
+ /**
+ * @param cluster Current cluster bit set.
+ * @param idx Node index.
+ */
+ void search(BitSet cluster, int idx) {
+ setBit(visitBitSet, idx);
+
+ cluster.set(idx);
+
+ ClusterNode node1 = nodes.get(idx);
+
+ for (int i = 0; i < nodeCnt; i++) {
+ if (i == idx || getBit(visitBitSet, i))
+ continue;
+
+ ClusterNode node2 = nodes.get(i);
+
+ boolean connected = ctx.connectionAvailable(node1, node2) ||
+ ctx.connectionAvailable(node2, node1);
+
+ if (connected)
+ search(cluster, i);
+ }
+ }
+
+ /**
+ * @param words Bit set words.
+ * @param bitIndex Bit index.
+ */
+ static void setBit(long words[], int bitIndex) {
+ int wordIndex = wordIndex(bitIndex);
+
+ words[wordIndex] |= (1L << bitIndex);
+ }
+
+ /**
+ * @param words Bit set words.
+ * @param bitIndex Bit index.
+ * @return Bit value.
+ */
+ static boolean getBit(long[] words, int bitIndex) {
+ int wordIndex = wordIndex(bitIndex);
+
+ return (words[wordIndex] & (1L << bitIndex)) != 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index 8c3c818..dbf5fbb 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -595,12 +595,22 @@ public class IgniteConfiguration {
warmupClos = cfg.getWarmupClosure();
}
+ /**
+ * TODO ZK
+ * @return
+ */
public CommunicationProblemResolver getCommunicationProblemResolver() {
return commProblemRslvr;
}
- public void setCommunicationProblemResolver(CommunicationProblemResolver commProblemRslvr) {
+ /**
+ * TODO ZK
+ * @param commProblemRslvr
+ */
+ public IgniteConfiguration setCommunicationProblemResolver(CommunicationProblemResolver commProblemRslvr) {
this.commProblemRslvr = commProblemRslvr;
+
+ return this;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index a641e6e..85bb7c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2696,6 +2696,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
objs.add(cfg.getGridLogger());
objs.add(cfg.getMBeanServer());
+ if (cfg.getCommunicationProblemResolver() != null)
+ objs.add(cfg.getCommunicationProblemResolver());
+
return objs;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 62ebcd8..8ae6313 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -71,6 +71,7 @@ import org.apache.ignite.configuration.PersistentStoreConfiguration;
import org.apache.ignite.configuration.TransactionConfiguration;
import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
import org.apache.ignite.internal.processors.igfs.IgfsThreadFactory;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
@@ -2270,6 +2271,8 @@ public class IgnitionEx {
initializeDefaultSpi(myCfg);
+ GridDiscoveryManager.initCommunicationErrorResolveConfiguration(myCfg);
+
initializeDefaultCacheConfiguration(myCfg);
ExecutorConfiguration[] execCfgs = myCfg.getExecutorConfiguration();
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index a151eb5..df84212 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -604,6 +604,14 @@ public abstract class GridManagerAdapter<T extends IgniteSpi> implements GridMan
return ctx.nodeAttributes();
}
+ @Override public boolean communicationErrorResolveSupported() {
+ return ctx.discovery().communicationErrorResolveSupported();
+ }
+
+ @Override public void resolveCommunicationError(ClusterNode node, Exception err) {
+ ctx.discovery().resolveCommunicationError(node, err);
+ }
+
/**
* @param e Exception to handle.
* @return GridSpiException Converted exception.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 97441d7..172615c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -53,8 +53,11 @@ import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationProblemResolver;
import org.apache.ignite.configuration.DataRegionConfiguration;
import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.DefaultCommunicationProblemResolver;
+import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.DiscoveryEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.internal.ClusterMetricsSnapshot;
@@ -108,6 +111,8 @@ import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.security.SecurityCredentials;
import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.spi.discovery.DiscoveryDataBag;
import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData;
import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
@@ -545,6 +550,9 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
});
}
+ if (ctx.config().getCommunicationProblemResolver() != null)
+ ctx.resource().injectGeneric(ctx.config().getCommunicationProblemResolver());
+
spi.setListener(new DiscoverySpiListener() {
private long gridStartTime;
@@ -2336,6 +2344,66 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
).start();
}
+ /**
+ * @param cfg Configuration.
+ * @throws IgniteCheckedException If configuration is not valid.
+ */
+ public static void initCommunicationErrorResolveConfiguration(IgniteConfiguration cfg) throws IgniteCheckedException {
+ CommunicationProblemResolver rslvr = cfg.getCommunicationProblemResolver();
+ CommunicationSpi commSpi = cfg.getCommunicationSpi();
+ DiscoverySpi discoverySpi = cfg.getDiscoverySpi();
+
+ if (rslvr != null) {
+ if (!supportsCommunicationErrorResolve(commSpi))
+ throw new IgniteCheckedException("CommunicationProblemResolver is configured, but communication SPI does not support communication" +
+ "problem resolve: " + commSpi.getClass().getName());
+
+ if (!supportsCommunicationErrorResolve(discoverySpi))
+ throw new IgniteCheckedException("CommunicationProblemResolver is configured, but discovery SPI does not support communication" +
+ "problem resolve: " + discoverySpi.getClass().getName());
+ }
+ else {
+ if (supportsCommunicationErrorResolve(commSpi) && supportsCommunicationErrorResolve(discoverySpi))
+ cfg.setCommunicationProblemResolver(new DefaultCommunicationProblemResolver());
+ }
+ }
+
+ /**
+ * @param spi Discovery SPI.
+ * @return {@code True} if SPI supports communication error resolve.
+ */
+ private static boolean supportsCommunicationErrorResolve(DiscoverySpi spi) {
+ return spi instanceof IgniteDiscoverySpi && ((IgniteDiscoverySpi)spi).supportsCommunicationErrorResolve();
+ }
+
+ /**
+ * @param spi Discovery SPI.
+ * @return {@code True} if SPI supports communication error resolve.
+ */
+ private static boolean supportsCommunicationErrorResolve(CommunicationSpi spi) {
+ return spi instanceof TcpCommunicationSpi;
+ }
+
+ /**
+ * @return {@code True} if communication error resolve is supported.
+ */
+ public boolean communicationErrorResolveSupported() {
+ return ctx.config().getCommunicationProblemResolver() != null;
+ }
+
+ /**
+ * @param node Problem node.
+ * @param err Error.
+ */
+ public void resolveCommunicationError(ClusterNode node, Exception err) {
+ DiscoverySpi spi = getSpi();
+
+ if (!supportsCommunicationErrorResolve(spi) || !supportsCommunicationErrorResolve(ctx.config().getCommunicationSpi()))
+ throw new UnsupportedOperationException();
+
+ ((IgniteDiscoverySpi)spi).resolveCommunicationError(node, err);
+ }
+
/** Worker for network segment checks. */
private class SegmentCheckWorker extends GridWorker {
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
index 9a1faa2..bf117f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
@@ -22,7 +22,7 @@ import org.apache.ignite.cluster.ClusterNode;
import org.apache.ignite.spi.discovery.DiscoverySpi;
/**
- * TODO ZK
+ *
*/
public interface IgniteDiscoverySpi extends DiscoverySpi {
/**
@@ -63,5 +63,5 @@ public interface IgniteDiscoverySpi extends DiscoverySpi {
* @param node Problem node.
* @param err Connection error.
*/
- public void onCommunicationConnectionError(ClusterNode node, Exception err);
+ public void resolveCommunicationError(ClusterNode node, Exception err);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 07ba214..50cf9fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -957,5 +957,15 @@ public abstract class IgniteSpiAdapter implements IgniteSpi {
@Override public Map<String, Object> nodeAttributes() {
return Collections.emptyMap();
}
+
+ /** {@inheritDoc} */
+ @Override public boolean communicationErrorResolveSupported() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolveCommunicationError(ClusterNode node, Exception err) {
+ throw new UnsupportedOperationException();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 96b3e61..f9c6ffd 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -358,4 +358,8 @@ public interface IgniteSpiContext {
* @return Current node attributes.
*/
public Map<String, Object> nodeAttributes();
+
+ public boolean communicationErrorResolveSupported();
+
+ public void resolveCommunicationError(ClusterNode node, Exception err);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 957a0e1..37be29f 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3409,22 +3409,16 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
boolean commErrResolve = false;
- if (connectionError(errs)) {
- DiscoverySpi discoverySpi = ignite.configuration().getDiscoverySpi();
+ IgniteSpiContext ctx = getSpiContext();
- if (discoverySpi instanceof IgniteDiscoverySpi) {
- IgniteDiscoverySpi discoverySpi0 = (IgniteDiscoverySpi)discoverySpi;
+ if (connectionError(errs) && ctx.communicationErrorResolveSupported()) {
+ commErrResolve = true;
- if (discoverySpi0.supportsCommunicationErrorResolve()) {
- commErrResolve = true;
-
- discoverySpi0.onCommunicationConnectionError(node, errs);
- }
- }
+ ctx.resolveCommunicationError(node, errs);
}
if (!commErrResolve && enableForcibleNodeKill) {
- if (getSpiContext().node(node.id()) != null
+ if (ctx.node(node.id()) != null
&& (CU.clientNode(node) || !CU.clientNode(getLocalNode())) &&
connectionError(errs)) {
String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
@@ -3435,7 +3429,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
else
U.warn(log, msg);
- getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+ ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
"rmtNode=" + node +
", errs=" + errs +
", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
deleted file mode 100644
index b2d4bf0..0000000
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DefaultCommunicationProblemResolver.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery;
-
-import java.util.BitSet;
-import java.util.List;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CommunicationProblemContext;
-import org.apache.ignite.configuration.CommunicationProblemResolver;
-
-/**
- *
- */
-public class DefaultCommunicationProblemResolver implements CommunicationProblemResolver {
- /** {@inheritDoc} */
- @Override public void resolve(CommunicationProblemContext ctx) {
- ClusterGraph graph = new ClusterGraph(ctx);
-
- BitSet cluster = graph.findLargestIndependentCluster();
-
- List<ClusterNode> nodes = ctx.topologySnapshot();
-
- if (graph.checkFullyConnected(cluster) && cluster.cardinality() < nodes.size()) {
- for (int i = 0; i < nodes.size(); i++) {
- if (!cluster.get(i))
- ctx.killNode(nodes.get(i));
- }
- }
- }
-
- /**
- *
- */
- private static class ClusterGraph {
- /** */
- private final static int WORD_IDX_SHIFT = 6;
-
- /**
- * @param bitIndex Bit index.
- * @return Word index containing bit with given index.
- */
- private static int wordIndex(int bitIndex) {
- return bitIndex >> WORD_IDX_SHIFT;
- }
-
- /** */
- private final int nodeCnt;
-
- /** */
- private final long[] visitBitSet;
-
- /** */
- private final CommunicationProblemContext ctx;
-
- /** */
- private final List<ClusterNode> nodes;
-
- ClusterGraph(CommunicationProblemContext ctx) {
- this.ctx = ctx;
-
- nodes = ctx.topologySnapshot();
-
- nodeCnt = nodes.size();
-
- assert nodeCnt > 0;
-
- visitBitSet = initBitSet(nodeCnt);
- }
-
- static long[] initBitSet(int bitCnt) {
- return new long[wordIndex(bitCnt - 1) + 1];
- }
-
- BitSet findLargestIndependentCluster() {
- BitSet maxCluster = null;
- int maxClusterSize = 0;
-
- for (int i = 0; i < nodeCnt; i++) {
- if (getBit(visitBitSet, i))
- continue;
-
- BitSet cluster = new BitSet(nodeCnt);
-
- search(cluster, i);
-
- int size = cluster.cardinality();
-
- if (maxCluster == null || size > maxClusterSize) {
- maxCluster = cluster;
- maxClusterSize = size;
- }
- }
-
- return maxCluster;
- }
-
- boolean checkFullyConnected(BitSet cluster) {
- int startIdx = 0;
-
- int clusterNodes = cluster.cardinality();
-
- for (;;) {
- int idx = cluster.nextSetBit(startIdx);
-
- if (idx == -1)
- break;
-
- ClusterNode node1 = nodes.get(idx);
-
- for (int i = 0; i < clusterNodes; i++) {
- if (!cluster.get(i) || i == idx)
- continue;
-
- ClusterNode node2 = nodes.get(i);
-
- if (cluster.get(i) && ctx.connectionAvailable(node1, node2))
- return false;
- }
-
- startIdx = idx + 1;
- }
-
- return true;
- }
-
- void search(BitSet cluster, int idx) {
- setBit(visitBitSet, idx);
-
- cluster.set(idx);
-
- ClusterNode node1 = nodes.get(idx);
-
- for (int i = 0; i < nodeCnt; i++) {
- if (i == idx || getBit(visitBitSet, i))
- continue;
-
- ClusterNode node2 = nodes.get(i);
-
- boolean connected = ctx.connectionAvailable(node1, node2) ||
- ctx.connectionAvailable(node2, node1);
-
- if (connected)
- search(cluster, i);
- }
- }
-
- static void setBit(long words[], int bitIndex) {
- int wordIndex = wordIndex(bitIndex);
-
- words[wordIndex] |= (1L << bitIndex);
- }
-
- static boolean getBit(long[] words, int bitIndex) {
- int wordIndex = wordIndex(bitIndex);
-
- return (words[wordIndex] & (1L << bitIndex)) != 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 0e2f851..ad8eca0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -2111,7 +2111,7 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscovery
}
/** {@inheritDoc} */
- @Override public void onCommunicationConnectionError(ClusterNode node, Exception err) {
+ @Override public void resolveCommunicationError(ClusterNode node, Exception err) {
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
index 98a22d7..14bb107 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
@@ -195,8 +195,8 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
}
/** {@inheritDoc} */
- @Override public void onCommunicationConnectionError(ClusterNode node, Exception err) {
- impl.onCommunicationConnectionError(node, err);
+ @Override public void resolveCommunicationError(ClusterNode node, Exception err) {
+ impl.resolveCommunicationError(node, err);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index 0074817..e64c801 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -148,7 +148,7 @@ class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implemen
* @param futPath Future path.
* @param nodes Nodes to ping.
*/
- void pingNodesAndNotifyFuture(final ZkRuntimeState rtState, final String futPath, List<ClusterNode> nodes) throws Exception {
+ void checkConnection(final ZkRuntimeState rtState, final String futPath, List<ClusterNode> nodes) {
final TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
IgniteFuture<BitSet> fut = spi.checkConnection(nodes);
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index ef3504f..f1ad869 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -49,6 +49,7 @@ import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.events.DiscoveryCustomEvent;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
import org.apache.ignite.internal.util.GridIntList;
@@ -247,17 +248,17 @@ public class ZookeeperDiscoveryImpl {
* @param node0 Problem node ID
* @param err Connect error.
*/
- public void onCommunicationConnectionError(ClusterNode node0, Exception err) {
- checkState();
-
+ public void resolveCommunicationError(ClusterNode node0, Exception err) {
ZookeeperClusterNode node = node(node0.id());
if (node == null)
- return;
+ throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
IgniteInternalFuture<Boolean> nodeStatusFut;
for (;;) {
+ checkState();
+
ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
if (fut == null || fut.isDone()) {
@@ -273,16 +274,16 @@ public class ZookeeperDiscoveryImpl {
", err= " + err + ']');
}
- ConnectionState connState;
+ try {
+ checkState();
+ }
+ catch (Exception e) {
+ fut.onError(e);
- synchronized (this) {
- connState = this.connState;
+ throw e;
}
- if (connState != ConnectionState.STARTED)
- fut.onError(new IgniteCheckedException("Node stopped."));
- else
- fut.scheduleCheckOnTimeout();
+ fut.scheduleCheckOnTimeout();
}
else
fut = commErrProcFut.get();
@@ -303,7 +304,8 @@ public class ZookeeperDiscoveryImpl {
}
try {
- nodeStatusFut.get();
+ if (!nodeStatusFut.get())
+ throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
}
catch (IgniteCheckedException e) {
throw new IgniteSpiException(e);
@@ -2240,7 +2242,7 @@ public class ZookeeperDiscoveryImpl {
runInWorkerThread(new ZkRunnable(rtState, this) {
@Override protected void run0() throws Exception {
- fut0.pingNodesAndNotifyFuture(rtState, futPath, rtState.commErrProcNodes);
+ fut0.checkConnection(rtState, futPath, rtState.commErrProcNodes);
}
});
}
@@ -2341,7 +2343,8 @@ public class ZookeeperDiscoveryImpl {
", rslvr=" + rslvr.getClass().getSimpleName() + ']');
}
- ZkCommunicationProblemContext ctx = new ZkCommunicationProblemContext(topSnapshot,
+ ZkCommunicationProblemContext ctx = new ZkCommunicationProblemContext(
+ topSnapshot,
initialNodes,
nodesRes);
@@ -2352,7 +2355,7 @@ public class ZookeeperDiscoveryImpl {
if (killedNodes != null) {
if (log.isInfoEnabled()) {
- log.info("Communication error resolver forces nodes stop [reqId=" + futId +
+ log.info("Communication error resolver forced nodes stop [reqId=" + futId +
", killNodeCnt=" + killedNodes.size() +
", nodeIds=" + U.nodeIds(killedNodes) + ']');
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
index 58098b9..44e48f9 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiBasicTest.java
@@ -61,6 +61,7 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.IgniteNodeAttributes;
import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
@@ -72,9 +73,11 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.logger.java.JavaLogger;
import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.IgniteSpiException;
import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
import org.apache.ignite.configuration.CommunicationProblemContext;
import org.apache.ignite.configuration.CommunicationProblemResolver;
@@ -147,7 +150,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
private boolean persistence;
/** */
- private CommunicationProblemResolver commProblemRslvr;
+ private IgniteOutClosure<CommunicationProblemResolver> commProblemRslvr;
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -248,7 +251,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
cfg.setCommunicationSpi(new ZkTestCommunicationSpi());
if (commProblemRslvr != null)
- cfg.setCommunicationProblemResolver(commProblemRslvr);
+ cfg.setCommunicationProblemResolver(commProblemRslvr.apply());
return cfg;
}
@@ -1808,7 +1811,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
assert nodes > 1;
sesTimeout = 2000;
- commProblemRslvr = new NoOpCommunicationProblemResolver();
+ commProblemRslvr = NoOpCommunicationProblemResolver.FACTORY;
startGridsMultiThreaded(nodes);
@@ -1828,7 +1831,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
ZookeeperDiscoverySpi spi = spi(ignite(idx1));
- spi.onCommunicationConnectionError(ignite(idx2).cluster().localNode(), new Exception("test"));
+ spi.resolveCommunicationError(ignite(idx2).cluster().localNode(), new Exception("test"));
checkInternalStructuresCleanup();
}
@@ -1840,7 +1843,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
public void testNoOpCommunicationErrorResolve_3() throws Exception {
// One node fails before sending communication status.
sesTimeout = 2000;
- commProblemRslvr = new NoOpCommunicationProblemResolver();
+ commProblemRslvr = NoOpCommunicationProblemResolver.FACTORY;
startGridsMultiThreaded(3);
@@ -1855,7 +1858,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
@Override public Object call() {
ZookeeperDiscoverySpi spi = spi(ignite(0));
- spi.onCommunicationConnectionError(ignite(1).cluster().localNode(), new Exception("test"));
+ spi.resolveCommunicationError(ignite(1).cluster().localNode(), new Exception("test"));
return null;
}
@@ -1883,11 +1886,11 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
* @throws Exception If failed.
*/
public void testNoOpCommunicationErrorResolve_4() throws Exception {
- // Coordinator changes while resolve process is in progress.
+ // Coordinator fails while resolve process is in progress.
testCommSpi = true;
sesTimeout = 2000;
- commProblemRslvr = new NoOpCommunicationProblemResolver();
+ commProblemRslvr = NoOpCommunicationProblemResolver.FACTORY;
startGrid(0);
@@ -1901,7 +1904,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
@Override public Object call() {
ZookeeperDiscoverySpi spi = spi(ignite(1));
- spi.onCommunicationConnectionError(ignite(2).cluster().localNode(), new Exception("test"));
+ spi.resolveCommunicationError(ignite(2).cluster().localNode(), new Exception("test"));
return null;
}
@@ -1970,6 +1973,8 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
}
/**
+ * TODO ZK: kill random, kill coordinator multiple times.
+ *
* @param startNodes Number of nodes to start.
* @param killNodes Nodes to kill by resolve process.
* @throws Exception If failed.
@@ -1977,7 +1982,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
private void communicationErrorResolve_KillNodes(int startNodes, Collection<Long> killNodes) throws Exception {
testCommSpi = true;
- commProblemRslvr = new TestNodeKillCommunicationProblemResolver(killNodes);
+ commProblemRslvr = TestNodeKillCommunicationProblemResolver.factory(killNodes);
startGrids(startNodes);
@@ -1986,20 +1991,28 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
commSpi.checkRes = new BitSet(startNodes);
ZookeeperDiscoverySpi spi = null;
+ UUID killNodeId = null;
for (Ignite node : G.allGrids()) {
ZookeeperDiscoverySpi spi0 = spi(node);
- if (!killNodes.contains(node.cluster().localNode().order())) {
+ if (!killNodes.contains(node.cluster().localNode().order()))
spi = spi0;
-
- break;
- }
+ else
+ killNodeId = node.cluster().localNode().id();
}
assertNotNull(spi);
+ assertNotNull(killNodeId);
- spi.onCommunicationConnectionError(ignite(1).cluster().localNode(), new Exception("test"));
+ try {
+ spi.resolveCommunicationError(spi.getNode(killNodeId), new Exception("test"));
+
+ fail("Exception is not thrown");
+ }
+ catch (IgniteSpiException e) {
+ assertTrue("Unexpected exception: " + e, e.getCause() instanceof ClusterTopologyCheckedException);
+ }
int expNodes = startNodes - killNodes.size();
@@ -2016,6 +2029,46 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testDefaultCommunicationErrorResolver1() throws Exception {
+ testCommSpi = true;
+ sesTimeout = 5000;
+
+ startGrids(3);
+
+ {
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(0));
+ commSpi.checkRes = new BitSet(3);
+ commSpi.checkRes.set(0);
+ commSpi.checkRes.set(1);
+ }
+ {
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(1));
+ commSpi.checkRes = new BitSet(3);
+ commSpi.checkRes.set(0);
+ commSpi.checkRes.set(1);
+ }
+ {
+ ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.forNode(ignite(2));
+ commSpi.checkRes = new BitSet(3);
+ commSpi.checkRes.set(2);
+ }
+
+ UUID killedId = nodeId(2);
+
+ assertNotNull(ignite(0).cluster().node(killedId));
+
+ ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+ spi.resolveCommunicationError(spi.getNode(ignite(1).cluster().localNode().id()), new Exception("test"));
+
+ waitForTopology(2);
+
+ assertNull(ignite(0).cluster().node(killedId));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testConnectionCheck() throws Exception {
final int NODES = 5;
@@ -2709,6 +2762,13 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
*
*/
static class NoOpCommunicationProblemResolver implements CommunicationProblemResolver {
+ /** */
+ static final IgniteOutClosure<CommunicationProblemResolver> FACTORY = new IgniteOutClosure<CommunicationProblemResolver>() {
+ @Override public CommunicationProblemResolver apply() {
+ return new NoOpCommunicationProblemResolver();
+ }
+ };
+
/** {@inheritDoc} */
@Override public void resolve(CommunicationProblemContext ctx) {
// No-op.
@@ -2719,6 +2779,18 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest {
*
*/
static class TestNodeKillCommunicationProblemResolver implements CommunicationProblemResolver {
+ /**
+ * @param killOrders Killed nodes order.
+ * @return Factory.
+ */
+ static IgniteOutClosure<CommunicationProblemResolver> factory(final Collection<Long> killOrders) {
+ return new IgniteOutClosure<CommunicationProblemResolver>() {
+ @Override public CommunicationProblemResolver apply() {
+ return new TestNodeKillCommunicationProblemResolver(killOrders);
+ }
+ };
+ }
+
/** */
final Collection<Long> killNodeOrders;
http://git-wip-us.apache.org/repos/asf/ignite/blob/4525b921/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index 93cd911..36403fa 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -607,6 +607,16 @@ public class GridSpiTestContext implements IgniteSpiContext {
return Collections.emptyMap();
}
+ /** {@inheritDoc} */
+ @Override public boolean communicationErrorResolveSupported() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resolveCommunicationError(ClusterNode node, Exception err) {
+ throw new UnsupportedOperationException();
+ }
+
/**
* @param cacheName Cache name.
* @return Map representing cache.