You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/07/24 15:57:06 UTC
[01/39] atlas git commit: ATLAS-1938: updated search UI to avoid
unnecessary calls on Refresh button click
Repository: atlas
Updated Branches:
refs/heads/feature-odf cac4bee57 -> 22780b933
ATLAS-1938: updated search UI to avoid unnecessary calls on Refresh button click
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/b1350182
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/b1350182
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/b1350182
Branch: refs/heads/feature-odf
Commit: b13501825028ea6c8ede8f6eacdf97bf58feacb8
Parents: ee8c81d
Author: Kalyani <ka...@freestoneinfotech.com>
Authored: Fri Jul 7 16:09:36 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jul 10 06:35:29 2017 -0700
----------------------------------------------------------------------
dashboardv2/public/js/views/search/SearchLayoutView.js | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/b1350182/dashboardv2/public/js/views/search/SearchLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchLayoutView.js b/dashboardv2/public/js/views/search/SearchLayoutView.js
index 7ac394c..c77d8a7 100644
--- a/dashboardv2/public/js/views/search/SearchLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchLayoutView.js
@@ -235,7 +235,9 @@ define(['require',
},
onRefreshButton: function() {
this.fetchCollection();
- if (this.searchVent) {
+ //to check url query param contain type or not
+ var checkURLValue = Utils.getUrlState.getQueryParams(this.url);
+ if (this.searchVent && (_.has(checkURLValue, "tag") || _.has(checkURLValue, "type") || _.has(checkURLValue, "query"))) {
this.searchVent.trigger('search:refresh');
}
},
[06/39] atlas git commit: ATLAS-1902: updated search UI to allow user
to select columns in search result
Posted by ma...@apache.org.
ATLAS-1902: updated search UI to allow user to select columns in search result
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/4e62a062
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/4e62a062
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/4e62a062
Branch: refs/heads/feature-odf
Commit: 4e62a0626c5946c7b81943669e859033884f919b
Parents: ea1c3b6
Author: kevalbhatt <kb...@apache.org>
Authored: Tue Jul 11 18:54:21 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 12 03:11:21 2017 -0700
----------------------------------------------------------------------
dashboardv2/gruntfile.js | 3 +
dashboardv2/package.json | 1 +
dashboardv2/public/css/scss/main.scss | 2 +-
dashboardv2/public/css/scss/override.scss | 22 +++-
dashboardv2/public/index.html | 1 +
dashboardv2/public/js/main.js | 4 +
.../public/js/utils/CommonViewFunction.js | 12 +-
dashboardv2/public/js/utils/TableLayout.js | 37 ++++--
.../js/views/audit/AuditTableLayoutView.js | 4 +-
.../views/audit/CreateAuditTableLayoutView.js | 4 +-
.../views/detail_page/DetailPageLayoutView.js | 4 +-
.../views/entity/EntityDetailTableLayoutView.js | 4 +-
.../public/js/views/search/SearchLayoutView.js | 20 ++-
.../js/views/search/SearchResultLayoutView.js | 124 ++++++++++++++++++-
14 files changed, 206 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/gruntfile.js
----------------------------------------------------------------------
diff --git a/dashboardv2/gruntfile.js b/dashboardv2/gruntfile.js
index c0e315f..1ca5069 100644
--- a/dashboardv2/gruntfile.js
+++ b/dashboardv2/gruntfile.js
@@ -119,6 +119,7 @@ module.exports = function(grunt) {
'backgrid-orderable-columns/js': 'backgrid-orderable-columns/backgrid-orderable-columns.js',
'backgrid-paginator/js': 'backgrid-paginator/backgrid-paginator.min.js',
'backgrid-sizeable-columns/js': 'backgrid-sizeable-columns/backgrid-sizeable-columns.js',
+ 'backgrid-columnmanager/js': 'backgrid-columnmanager/src/Backgrid.ColumnManager.js',
'jquery-asBreadcrumbs/js': 'jquery-asBreadcrumbs/dist/jquery-asBreadcrumbs.min.js',
'd3': 'd3/d3.min.js',
'd3/': 'd3-tip/index.js',
@@ -144,6 +145,7 @@ module.exports = function(grunt) {
'backgrid-orderable-columns/css': 'backgrid-orderable-columns/backgrid-orderable-columns.css',
'backgrid-paginator/css': 'backgrid-paginator/backgrid-paginator.css',
'backgrid-sizeable-columns/css': 'backgrid-sizeable-columns/backgrid-sizeable-columns.css',
+ 'backgrid-columnmanager/css': 'backgrid-columnmanager/lib/Backgrid.ColumnManager.css',
'jquery-asBreadcrumbs/css': 'jquery-asBreadcrumbs/dist/css/asBreadcrumbs.min.css',
'select2/css': 'select2/dist/css/select2.min.css',
'backgrid-select-all': 'backgrid-select-all/backgrid-select-all.min.css',
@@ -172,6 +174,7 @@ module.exports = function(grunt) {
'backgrid-orderable-columns': 'backgrid-orderable-columns/LICENSE-MIT',
'backgrid-paginator': 'backgrid-paginator/LICENSE-MIT',
'backgrid-sizeable-columns': 'backgrid-sizeable-columns/LICENSE-MIT',
+ 'backgrid-columnmanager': 'backgrid-columnmanager/LICENSE',
'jquery-asBreadcrumbs': 'jquery-asBreadcrumbs/LICENSE',
'd3': 'd3/LICENSE',
'd3/': 'd3-tip/LICENSE',
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/package.json
----------------------------------------------------------------------
diff --git a/dashboardv2/package.json b/dashboardv2/package.json
index 5be682e..bcf5fe7 100644
--- a/dashboardv2/package.json
+++ b/dashboardv2/package.json
@@ -21,6 +21,7 @@
"backbone.paginator": "2.0.5",
"backbone.wreqr": "1.4.0",
"backgrid": "0.3.8",
+ "backgrid-columnmanager": "0.2.4",
"backgrid-filter": "0.3.7",
"backgrid-orderable-columns": "0.1.2",
"backgrid-paginator": "0.3.9",
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/css/scss/main.scss
----------------------------------------------------------------------
diff --git a/dashboardv2/public/css/scss/main.scss b/dashboardv2/public/css/scss/main.scss
index a608c95..83c2a5a 100644
--- a/dashboardv2/public/css/scss/main.scss
+++ b/dashboardv2/public/css/scss/main.scss
@@ -199,7 +199,7 @@ ul {
}
.termTableBreadcrumb {
- min-width: 200px;
+ /* min-width: 200px; */
/* width: 200px; */
/* display: inline-block; */
i {
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/css/scss/override.scss
----------------------------------------------------------------------
diff --git a/dashboardv2/public/css/scss/override.scss b/dashboardv2/public/css/scss/override.scss
index 6821d77..d2a4bec 100644
--- a/dashboardv2/public/css/scss/override.scss
+++ b/dashboardv2/public/css/scss/override.scss
@@ -76,7 +76,7 @@
border-color: $color_jungle_green_approx;
border-bottom-style: solid;
box-shadow: none;
- padding: 20px 30px;
+ padding: 20px 15px;
background-color: transparent;
text-align: left;
font-weight: 800;
@@ -89,7 +89,7 @@
border-color: $color_gallery_approx;
color: #333333;
font-weight: 100;
- padding: 15px 30px;
+ padding: 10px 15px;
}
.backgrid-paginator ul > li > a:hover,
@@ -286,3 +286,21 @@
.query-builder .rule-value-container input {
padding: 6px 12px !important;
}
+
+div.columnmanager-visibilitycontrol {
+ width: auto;
+ margin-bottom: 5px;
+}
+
+.columnmanager-dropdown-container > li > span.column-label {
+ width: auto;
+}
+
+div.columnmanager-dropdown-container.open {
+ overflow: auto;
+}
+
+.columnmanager-visibilitycontrol.open .btn-atlasAction {
+ background-color: #37bb9b;
+ color: #fff;
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/index.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/index.html b/dashboardv2/public/index.html
index b460686..2b0b5d7 100644
--- a/dashboardv2/public/index.html
+++ b/dashboardv2/public/index.html
@@ -42,6 +42,7 @@
<link rel="stylesheet" href="js/libs/backgrid-paginator/css/backgrid-paginator.css">
<link rel="stylesheet" href="js/libs/backgrid-orderable-columns/css/backgrid-orderable-columns.css">
<link rel="stylesheet" href="js/libs/backgrid-sizeable-columns/css/backgrid-sizeable-columns.css">
+ <link rel="stylesheet" href="js/libs/backgrid-columnmanager/css/Backgrid.ColumnManager.css">
<link rel="stylesheet" href="js/libs/select2/css/select2.min.css">
<link rel="stylesheet" href="js/libs/bootstrap/css/bootstrap.min.css">
<link rel="stylesheet" href="js/libs/jquery-asBreadcrumbs/css/asBreadcrumbs.min.css">
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/main.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/main.js b/dashboardv2/public/js/main.js
index 9235d28..b04141f 100644
--- a/dashboardv2/public/js/main.js
+++ b/dashboardv2/public/js/main.js
@@ -84,6 +84,9 @@ require.config({
'backgrid-select-all': {
'deps': ['backbone', 'backgrid']
},
+ 'backgrid-columnmanager': {
+ 'deps': ['backbone', 'backgrid'],
+ },
'hbs': {
'deps': ['underscore', 'handlebars']
},
@@ -126,6 +129,7 @@ require.config({
'backgrid-orderable': 'libs/backgrid-orderable-columns/js/backgrid-orderable-columns',
'backgrid-paginator': 'libs/backgrid-paginator/js/backgrid-paginator.min',
'backgrid-sizeable': 'libs/backgrid-sizeable-columns/js/backgrid-sizeable-columns',
+ 'backgrid-columnmanager': 'libs/backgrid-columnmanager/js/Backgrid.ColumnManager',
'asBreadcrumbs': 'libs/jquery-asBreadcrumbs/js/jquery-asBreadcrumbs.min',
'd3': 'libs/d3/d3.min',
'd3-tip': 'libs/d3/index',
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/utils/CommonViewFunction.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/utils/CommonViewFunction.js b/dashboardv2/public/js/utils/CommonViewFunction.js
index de46667..a04fb3f 100644
--- a/dashboardv2/public/js/utils/CommonViewFunction.js
+++ b/dashboardv2/public/js/utils/CommonViewFunction.js
@@ -92,7 +92,8 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
var scope = options.scope,
valueObject = options.valueObject,
extractJSON = options.extractJSON,
- entityDef = options.entityDef;
+ isTable = _.isUndefined(options.isTable) ? true : options.isTable,
+ attributeDefs = options.attributeDefs;
var table = "",
fetchInputOutputValue = function(id) {
@@ -212,7 +213,7 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
_.sortBy(_.keys(valueObject)).map(function(key) {
key = _.escape(key);
var keyValue = valueObject[key];
- var defEntity = _.find(entityDef, { name: key });
+ var defEntity = _.find(attributeDefs, { name: key });
if (defEntity && defEntity.typeName) {
var defEntityType = defEntity.typeName.toLocaleLowerCase();
if (defEntityType === 'date' || defEntityType === 'time') {
@@ -225,7 +226,12 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
keyValue = extractObject(keyValue)
}
}
- table += '<tr><td>' + _.escape(key) + '</td><td>' + (_.isObject(valueObject[key]) ? keyValue : _.escape(keyValue)) + '</td></tr>';
+ if (isTable) {
+ table += '<tr><td>' + _.escape(key) + '</td><td>' + (_.isObject(valueObject[key]) ? keyValue : _.escape(keyValue)) + '</td></tr>';
+ } else {
+ table += '<div>' + (_.isObject(valueObject[key]) ? keyValue : _.escape(keyValue)) + '</div>';
+ }
+
});
return table;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/utils/TableLayout.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/utils/TableLayout.js b/dashboardv2/public/js/utils/TableLayout.js
index f6183a9..db0a9da 100644
--- a/dashboardv2/public/js/utils/TableLayout.js
+++ b/dashboardv2/public/js/utils/TableLayout.js
@@ -26,7 +26,8 @@ define(['require',
'backgrid-paginator',
'backgrid-sizeable',
'backgrid-orderable',
- 'backgrid-select-all'
+ 'backgrid-select-all',
+ 'backgrid-columnmanager'
], function(require, Backbone, FSTablelayoutTmpl) {
'use strict';
@@ -103,10 +104,13 @@ define(['require',
}
},
columnOpts: {
- initialColumnsVisible: 4,
- // State settings
- saveState: false,
- loadStateOnInit: true
+ opts: {
+ initialColumnsVisible: 4,
+ // State settings
+ saveState: false,
+ loadStateOnInit: true
+ },
+ visibilityControlOpts: {}
},
includePagination: true,
@@ -301,14 +305,20 @@ define(['require',
* ColumnManager for the table
*/
renderColumnManager: function() {
- var $el = this.$("[data-id='control']");
- var colManager = new Backgrid.Extension.ColumnManager(this.columns, this.columnOpts);
- // Add control
- var colVisibilityControl = new Backgrid.Extension.ColumnManagerVisibilityControl({
- columnManager: colManager
- });
-
+ var that = this,
+ $el = this.$("[data-id='control']"),
+ colManager = new Backgrid.Extension.ColumnManager(this.columns, this.columnOpts.opts),
+ // Add control
+ colVisibilityControl = new Backgrid.Extension.ColumnManagerVisibilityControl(_.extend({
+ columnManager: colManager,
+ }, this.columnOpts.visibilityControlOpts));
$el.append(colVisibilityControl.render().el);
+ colManager.on("state-changed", function(state) {
+ that.collection.trigger("state-changed", state);
+ });
+ colManager.on("state-saved", function() {
+ that.collection.trigger("state-changed");
+ });
},
renderSizeAbleColumns: function() {
@@ -323,7 +333,7 @@ define(['require',
// Add resize handlers
var sizeHandler = new Backgrid.Extension.SizeAbleColumnsHandlers({
sizeAbleColumns: sizeAbleCol,
- grid: this.getGridObj(),
+ // grid: this.getGridObj(),
saveModelWidth: true
});
this.$('thead').before(sizeHandler.render().el);
@@ -341,6 +351,7 @@ define(['require',
grid: this.getGridObj(),
columns: this.columns
});
+ this.$('thead').before(sizeAbleCol.render().el);
var orderHandler = new Backgrid.Extension.OrderableColumns({
grid: this.getGridObj(),
sizeAbleColumns: sizeAbleCol
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/views/audit/AuditTableLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/audit/AuditTableLayoutView.js b/dashboardv2/public/js/views/audit/AuditTableLayoutView.js
index f650b1c..223e728 100644
--- a/dashboardv2/public/js/views/audit/AuditTableLayoutView.js
+++ b/dashboardv2/public/js/views/audit/AuditTableLayoutView.js
@@ -57,7 +57,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'guid', 'entity', 'entityName', 'entityDef'));
+ _.extend(this, _.pick(options, 'guid', 'entity', 'entityName', 'attributeDefs'));
this.entityCollection = new VEntityList();
this.limit = 26;
this.entityCollection.url = UrlLinks.entityCollectionaudit(this.guid);
@@ -239,7 +239,7 @@ define(['require',
that.action = $(e.target).data("action");
var eventModel = that.entityCollection.fullCollection.findWhere({ 'eventKey': $(e.currentTarget).data('modalid') }).toJSON(),
collectionModel = new that.entityCollection.model(eventModel),
- view = new CreateAuditTableLayoutView({ guid: that.guid, entityModel: collectionModel, action: that.action, entity: that.entity, entityName: that.entityName, entityDef: that.entityDef });
+ view = new CreateAuditTableLayoutView({ guid: that.guid, entityModel: collectionModel, action: that.action, entity: that.entity, entityName: that.entityName, attributeDefs: that.attributeDefs });
var modal = new Modal({
title: that.action,
content: view,
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/views/audit/CreateAuditTableLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/audit/CreateAuditTableLayoutView.js b/dashboardv2/public/js/views/audit/CreateAuditTableLayoutView.js
index 34c56cb..71332d7 100644
--- a/dashboardv2/public/js/views/audit/CreateAuditTableLayoutView.js
+++ b/dashboardv2/public/js/views/audit/CreateAuditTableLayoutView.js
@@ -55,7 +55,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'guid', 'entityModel', 'action', 'entity', 'entityName', 'entityDef'));
+ _.extend(this, _.pick(options, 'guid', 'entityModel', 'action', 'entity', 'entityName', 'attributeDefs'));
},
bindEvents: function() {},
onRender: function() {
@@ -84,7 +84,7 @@ define(['require',
var tagHeader = ((name ? name : this.entityName));
this.ui.tagHeader.append(tagHeader);
this.ui.auditHeaderValue.html('<th>Key</th><th>New Value</th>');
- table = CommonViewFunction.propertyTable({ scope: this, valueObject: values, entityDef: this.entityDef, extractJSON: { extractKey: 'value' } });
+ table = CommonViewFunction.propertyTable({ scope: this, valueObject: values, attributeDefs: this.attributeDefs, extractJSON: { extractKey: 'value' } });
if (table.length) {
this.ui.noData.hide();
this.ui.tableAudit.show();
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js b/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
index d44962f..c45755f 100644
--- a/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
+++ b/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
@@ -242,12 +242,12 @@ define(['require',
},
getEntityDef: function(obj) {
var data = this.entityDefCollection.fullCollection.findWhere({ name: obj.entity.typeName }).toJSON();
- var entityDef = Utils.getNestedSuperTypeObj({
+ var attributeDefs = Utils.getNestedSuperTypeObj({
data: data,
attrMerge: true,
collection: this.entityDefCollection
});
- obj['entityDef'] = entityDef;
+ obj['attributeDefs'] = attributeDefs;
this.renderEntityDetailTableLayoutView(obj);
this.renderAuditTableLayoutView(obj);
},
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js b/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
index 79f7807..87c5dcd 100644
--- a/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
+++ b/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
@@ -48,7 +48,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'entity', 'referredEntities', 'typeHeaders', 'entityDef'));
+ _.extend(this, _.pick(options, 'entity', 'referredEntities', 'typeHeaders', 'attributeDefs'));
this.entityModel = new VEntity({});
},
bindEvents: function() {},
@@ -65,7 +65,7 @@ define(['require',
});
attributeObject.columns = valueSorted;
}
- var table = CommonViewFunction.propertyTable({ scope: this, valueObject: attributeObject, entityDef: this.entityDef });
+ var table = CommonViewFunction.propertyTable({ scope: this, valueObject: attributeObject, attributeDefs: this.attributeDefs });
that.ui.detailValue.append(table);
}
});
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/views/search/SearchLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchLayoutView.js b/dashboardv2/public/js/views/search/SearchLayoutView.js
index c77d8a7..cb8844d 100644
--- a/dashboardv2/public/js/views/search/SearchLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchLayoutView.js
@@ -403,16 +403,28 @@ define(['require',
},
triggerSearch: function(value) {
this.query[this.type].query = value || null;
+ var params = {
+ searchType: this.type,
+ dslChecked: this.ui.searchType.is(':checked')
+ }
this.query[this.type].type = this.ui.typeLov.select2('val') || null;
if (!this.dsl) {
this.query[this.type].tag = this.ui.tagLov.select2('val') || null;
}
+ if (this.dsl) {
+ params['attributes'] = null;
+ } else {
+ var columnList = JSON.parse(Utils.localStorage.getValue('columnList'));
+ if (columnList) {
+ params['attributes'] = columnList[this.query[this.type].type];
+ } else {
+ params['attributes'] = null;
+ }
+ }
+
Utils.setUrl({
url: '#!/search/searchResult',
- urlParams: _.extend(this.query[this.type], {
- searchType: this.type,
- dslChecked: this.ui.searchType.is(':checked')
- }),
+ urlParams: _.extend(this.query[this.type], params),
updateTabState: function() {
return { searchUrl: this.url, stateChanged: true };
},
http://git-wip-us.apache.org/repos/asf/atlas/blob/4e62a062/dashboardv2/public/js/views/search/SearchResultLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchResultLayoutView.js b/dashboardv2/public/js/views/search/SearchResultLayoutView.js
index 5236751..dba27fa 100644
--- a/dashboardv2/public/js/views/search/SearchResultLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchResultLayoutView.js
@@ -136,13 +136,22 @@ define(['require',
this.limit = 25;
this.asyncFetchCounter = 0;
this.offset = 0;
+ this.columnToShow = this.value && this.value.attributes ? this.value.attributes.split(',') : [];
this.commonTableOptions = {
collection: this.searchCollection,
- includeFilter: false,
includePagination: false,
- includePageSize: false,
includeFooterRecords: false,
- includeSizeAbleColumns: false,
+ includeColumnManager: (this.value && this.value.searchType === "basic" ? true : false),
+ includeOrderAbleColumns: false,
+ columnOpts: {
+ opts: {
+ initialColumnsVisible: null,
+ saveState: false
+ },
+ visibilityControlOpts: {
+ buttonTemplate: _.template("<button class='btn btn-atlasAction btn-atlas'>Columns <i class='fa fa-caret-down'></i></button>")
+ }
+ },
gridOpts: {
emptyText: 'No Record found!',
className: 'table table-hover backgrid table-quickMenu'
@@ -206,6 +215,25 @@ define(['require',
}
}
}, this);
+ this.listenTo(this.searchCollection, "state-changed", function(state) {
+ if (Utils.getUrlState.isSearchTab()) {
+ this.updateColumnList(state);
+ var columnList = JSON.parse(Utils.localStorage.getValue('columnList'));
+ if (!columnList && this.value.type) {
+ columnList = {};
+ columnList[this.value.type] = this.value.attributes;
+ } else {
+ columnList[this.value.type] = this.value.attributes;
+ }
+ Utils.localStorage.setValue('columnList', JSON.stringify(columnList));
+ Utils.setUrl({
+ url: '#!/search/searchResult',
+ urlParams: this.value,
+ mergeBrowserUrl: false,
+ trigger: true
+ });
+ }
+ }, this);
this.listenTo(this.searchVent, "search:refresh", function(model, response) {
this.fetchCollection();
}, this);
@@ -225,6 +253,7 @@ define(['require',
'searchType': 'basic'
};
}
+ this.updateColumnList();
this.fetchCollection(value);
$('body').click(function(e) {
var iconEvnt = e.target.nodeName;
@@ -242,6 +271,22 @@ define(['require',
}
this.showHideFilter();
},
+ updateColumnList: function(updatedList) {
+ if (updatedList) {
+ var listOfColumns = []
+ _.map(updatedList, function(obj) {
+ var key = obj.name;
+ if (key == "selected" || key == "displayText" || key == "description" || key == "typeName" || key == "owner" || key == "tag" || key == "terms") {
+ return;
+ }
+ if (obj.renderable) {
+ listOfColumns.push(obj.name);
+ }
+ });
+ this.value.attributes = listOfColumns.length ? listOfColumns.join(",") : null;
+ }
+ this.columnToShow = this.value && this.value.attributes ? this.value.attributes.split(',') : [];
+ },
generateQueryOfFilter: function() {
var value = this.value,
entityFilters = this.filterObj && this.filterObj.entityFilters ? this.filterObj.entityFilters[value.type] : null,
@@ -292,7 +337,8 @@ define(['require',
entityFilters = this.filterObj && this.filterObj.entityFilters ? this.filterObj.entityFilters[this.value.type] : null,
filterObj = {
'entityFilters': entityFilters ? entityFilters.result : null,
- 'tagFilters': tagFilters ? tagFilters.result : null
+ 'tagFilters': tagFilters ? tagFilters.result : null,
+ 'attributes': this.columnToShow.length ? this.columnToShow : null
}
this.showLoader();
if (Globals.searchApiCallRef && Globals.searchApiCallRef.readyState === 1) {
@@ -393,7 +439,21 @@ define(['require',
var that = this,
count = 5;
require(['utils/TableLayout'], function(TableLayout) {
- var columns = new Backgrid.Columns(that.getFixedDslColumn());
+ var columnCollection = Backgrid.Columns.extend({
+ sortKey: "position",
+ comparator: function(item) {
+ return item.get(this.sortKey) || 999;
+ },
+ setPositions: function() {
+ _.each(this.models, function(model, index) {
+ model.set("position", index + 1, { silent: true });
+ });
+ return this;
+ }
+ });
+ var columns = new columnCollection(that.getFixedDslColumn());
+ columns.setPositions().sort();
+ //var columns = new Backgrid.Columns(that.getFixedDslColumn());
that.REntityTableLayoutView.show(new TableLayout(_.extend({}, that.commonTableOptions, {
columns: columns
})));
@@ -438,6 +498,9 @@ define(['require',
cell: "html",
editable: false,
sortable: false,
+ resizeable: true,
+ orderable: true,
+ renderable: true,
className: "searchTableName",
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
@@ -473,6 +536,9 @@ define(['require',
cell: "String",
editable: false,
sortable: false,
+ resizeable: true,
+ orderable: true,
+ renderable: true,
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
var obj = model.toJSON();
@@ -487,6 +553,9 @@ define(['require',
cell: "Html",
editable: false,
sortable: false,
+ resizeable: true,
+ orderable: true,
+ renderable: true,
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
var obj = model.toJSON();
@@ -501,6 +570,9 @@ define(['require',
cell: "String",
editable: false,
sortable: false,
+ resizeable: true,
+ orderable: true,
+ renderable: true,
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
var obj = model.toJSON();
@@ -515,7 +587,9 @@ define(['require',
cell: "Html",
editable: false,
sortable: false,
+ resizeable: true,
orderable: true,
+ renderable: true,
className: 'searchTag',
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
@@ -535,7 +609,9 @@ define(['require',
cell: "Html",
editable: false,
sortable: false,
+ resizeable: true,
orderable: true,
+ renderable: true,
className: 'searchTerm',
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
@@ -553,6 +629,44 @@ define(['require',
})
};
}
+ if (this.value && this.value.searchType === "basic") {
+ var def = this.entityDefCollection.fullCollection.find({ name: this.value.type });
+ if (def) {
+ var attrObj = Utils.getNestedSuperTypeObj({ data: def.toJSON(), collection: this.entityDefCollection, attrMerge: true });
+ _.each(attrObj, function(obj, key) {
+ var key = obj.name,
+ isEenderable = that.columnToShow.length ? _.contains(that.columnToShow, key) : false;
+ if (key == "name" || key == "description" || key == "owner") {
+ return;
+ }
+ col[obj.name] = {
+ label: obj.name.capitalize(),
+ cell: "Html",
+ editable: false,
+ sortable: false,
+ resizeable: true,
+ orderable: true,
+ renderable: isEenderable,
+ formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
+ fromRaw: function(rawValue, model) {
+ var modelObj = model.toJSON();
+
+ if (modelObj && modelObj.attributes && !_.isUndefined(modelObj.attributes[key])) {
+ var tempObj = {
+ 'scope': that,
+ 'attributeDefs': [obj],
+ 'valueObject': {},
+ 'isTable': false
+ }
+ tempObj.valueObject[key] = modelObj.attributes[key]
+ return CommonViewFunction.propertyTable(tempObj);
+ }
+ }
+ })
+ };
+ });
+ }
+ }
return this.searchCollection.constructor.getTableCols(col, this.searchCollection);
},
addTagModalView: function(guid, multiple) {
[36/39] atlas git commit: ATLAS-1960: fixed handling of import/export
APIs in HA configuration
Posted by ma...@apache.org.
ATLAS-1960: fixed handling of import/export APIs in HA configuration
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/d23d61b2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/d23d61b2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/d23d61b2
Branch: refs/heads/feature-odf
Commit: d23d61b2a88a472978dcd15428a61dfa48d31215
Parents: e0d2cdc
Author: ashutoshm <am...@hortonworks.com>
Authored: Fri Jul 21 12:17:42 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 21 19:24:42 2017 -0700
----------------------------------------------------------------------
.../atlas/web/filters/ActiveServerFilter.java | 15 ++++++++-
.../web/filters/ActiveServerFilterTest.java | 35 ++++++++++++++++----
2 files changed, 42 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/d23d61b2/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java
index 36f2688..29d829f 100644
--- a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java
+++ b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java
@@ -100,10 +100,23 @@ public class ActiveServerFilter implements Filter {
}
}
+ final String adminUriNotFiltered[] = { "/admin/export", "/admin/import", "/admin/importfile" };
private boolean isFilteredURI(ServletRequest servletRequest) {
HttpServletRequest httpServletRequest = (HttpServletRequest) servletRequest;
String requestURI = httpServletRequest.getRequestURI();
- return requestURI.contains("/admin/");
+
+ if(requestURI.contains("/admin/")) {
+ for (String s : adminUriNotFiltered) {
+ if (requestURI.contains(s)) {
+ LOG.error("URL not supported in HA mode: {}", requestURI);
+ return false;
+ }
+ }
+
+ return true;
+ } else {
+ return false;
+ }
}
boolean isInstanceActive() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/d23d61b2/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java b/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java
index a8d1110..3396072 100644
--- a/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java
+++ b/webapp/src/test/java/org/apache/atlas/web/filters/ActiveServerFilterTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -99,7 +99,28 @@ public class ActiveServerFilterTest {
activeServerFilter.doFilter(servletRequest, servletResponse, filterChain);
- verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS+"types");
+ verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "types");
+ }
+
+
+ @Test
+ public void adminImportRequestsToPassiveServerShouldToActiveServerAddress() throws IOException, ServletException {
+ String importExportUrls[] = {"api/admin/export", "api/admin/import", "api/admin/importfile"};
+
+ for (String partialUrl : importExportUrls) {
+ when(serviceState.getState()).thenReturn(ServiceState.ServiceStateValue.PASSIVE);
+ when(servletRequest.getRequestURI()).thenReturn(partialUrl);
+
+ ActiveServerFilter activeServerFilter = new ActiveServerFilter(activeInstanceState, serviceState);
+
+ when(activeInstanceState.getActiveServerAddress()).thenReturn(ACTIVE_SERVER_ADDRESS);
+ when(servletRequest.getRequestURI()).thenReturn(partialUrl);
+ when(servletRequest.getMethod()).thenReturn(HttpMethod.GET);
+
+ activeServerFilter.doFilter(servletRequest, servletResponse, filterChain);
+
+ verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + partialUrl);
+ }
}
@Test
@@ -116,7 +137,7 @@ public class ActiveServerFilterTest {
activeServerFilter.doFilter(servletRequest, servletResponse, filterChain);
- verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS+"types?query=TRAIT");
+ verify(servletResponse).sendRedirect(ACTIVE_SERVER_ADDRESS + "types?query=TRAIT");
}
@@ -133,7 +154,7 @@ public class ActiveServerFilterTest {
activeServerFilter.doFilter(servletRequest, servletResponse, filterChain);
- verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS+"types");
+ verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS + "types");
verify(servletResponse).setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
}
@@ -150,7 +171,7 @@ public class ActiveServerFilterTest {
activeServerFilter.doFilter(servletRequest, servletResponse, filterChain);
- verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS+"types");
+ verify(servletResponse).setHeader("Location", ACTIVE_SERVER_ADDRESS + "types");
verify(servletResponse).setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
}
[22/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering
Posted by ma...@apache.org.
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/377fe19f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/377fe19f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/377fe19f
Branch: refs/heads/feature-odf
Commit: 377fe19f023a4036f9fa1503dec6ea698fdb214b
Parents: 71f9c06
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Tue Jul 18 00:25:36 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 18 08:05:53 2017 -0700
----------------------------------------------------------------------
.../atlas/discovery/EntitySearchProcessor.java | 73 ++++++++++----------
.../apache/atlas/discovery/SearchContext.java | 2 +-
.../apache/atlas/discovery/SearchProcessor.java | 14 +++-
3 files changed, 50 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 605cb15..3204ecf 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
@@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor {
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor");
private final AtlasIndexQuery indexQuery;
- private final AtlasGraphQuery partialGraphQuery;
- private final AtlasGraphQuery allGraphQuery;
+ private final AtlasGraphQuery graphQuery;
+ private final AtlasGraphQuery filterGraphQuery;
public EntitySearchProcessor(SearchContext context) {
super(context);
- AtlasEntityType entityType = context.getEntityType();
- AtlasClassificationType classificationType = context.getClassificationType();
- FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
- Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- Set<String> solrAttributes = new HashSet<>();
- Set<String> gremlinAttributes = new HashSet<>();
- Set<String> allAttributes = new HashSet<>();
+ final AtlasEntityType entityType = context.getEntityType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ final Set<String> solrAttributes = new HashSet<>();
+ final Set<String> gremlinAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
- boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY && canApplySolrFilter(entityType, filterCriteria, false);
+ final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+ final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
- if (useSolrSearch) {
- StringBuilder solrQuery = new StringBuilder();
+ StringBuilder solrQuery = new StringBuilder();
+ if (typeSearchBySolr) {
constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ }
+
+ if (attrSearchBySolr) {
constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes);
+ } else {
+ gremlinAttributes.addAll(solrAttributes);
+ }
+ if (solrQuery.length() > 0) {
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
+ } else {
+ indexQuery = null;
+ }
- if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null) {
- AtlasGraphQuery query = context.getGraph().query();
+ if (CollectionUtils.isNotEmpty(gremlinAttributes) || !typeSearchBySolr) {
+ AtlasGraphQuery query = context.getGraph().query();
- addClassificationNameConditionIfNecessary(query);
+ if (!typeSearchBySolr) {
+ query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+ }
+
+ graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
- partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
- } else {
- partialGraphQuery = null;
+ if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) {
+ graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
}
} else {
- indexQuery = null;
- partialGraphQuery = null;
+ graphQuery = null;
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
- addClassificationNameConditionIfNecessary(query);
-
- allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
+ filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities()) {
- allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
+ filterGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
}
}
@@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor {
vertices = getVerticesFromIndexQueryResult(queryResult);
- if (partialGraphQuery != null) {
+ if (graphQuery != null) {
AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices));
- guidQuery.addConditionsFrom(partialGraphQuery);
+ guidQuery.addConditionsFrom(graphQuery);
vertices = getVertices(guidQuery.vertices().iterator());
}
} else {
- Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
+ Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset, limit).iterator();
if (!queryResult.hasNext()) { // no more results from query - end of search
break;
@@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
- query.addConditionsFrom(allGraphQuery);
+ query.addConditionsFrom(filterGraphQuery);
List<AtlasVertex> ret = getVertices(query.vertices().iterator());
@@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor {
return ret;
}
-
- private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) {
- if (context.getClassificationType() != null && !context.needClassificationProcessor()) {
- query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes());
- }
- }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 55a07f3..8dd7667 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -109,7 +109,7 @@ public class SearchContext {
}
public boolean needClassificationProcessor() {
- return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters()) || entityType == null);
+ return classificationType != null;
}
public boolean needEntityProcessor() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 1481809..ff0bd2e 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -88,7 +88,7 @@ public abstract class SearchProcessor {
public abstract List<AtlasVertex> execute();
public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
- return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices);
+ return nextProcessor == null || CollectionUtils.isEmpty(entityVertices) ? entityVertices : nextProcessor.filter(entityVertices);
}
@@ -193,12 +193,20 @@ public abstract class SearchProcessor {
String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) {
- solrQuery.append(AND_STR).append(filterQuery);
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append(filterQuery);
}
}
if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
- solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE");
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append("v.\"__state\":").append("ACTIVE");
}
}
[21/39] atlas git commit: ATLAS-1927: UI updates in rendering of tag
attributes
Posted by ma...@apache.org.
ATLAS-1927: UI updates in rendering of tag attributes
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/71f9c065
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/71f9c065
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/71f9c065
Branch: refs/heads/feature-odf
Commit: 71f9c065225405d188c622a6898686ee5a73817f
Parents: d7ce0a5
Author: Kalyani <ka...@freestoneinfotech.com>
Authored: Fri Jul 14 15:14:14 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 18 08:01:20 2017 -0700
----------------------------------------------------------------------
dashboardv2/public/css/scss/tag.scss | 40 +++++++++++++++++---
.../views/detail_page/DetailPageLayoutView.js | 5 ++-
.../js/views/tag/TagDetailTableLayoutView.js | 31 +++++++++------
.../public/js/views/tag/addTagModalView.js | 13 +++++--
4 files changed, 67 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/71f9c065/dashboardv2/public/css/scss/tag.scss
----------------------------------------------------------------------
diff --git a/dashboardv2/public/css/scss/tag.scss b/dashboardv2/public/css/scss/tag.scss
index fc0652b..39bc995 100644
--- a/dashboardv2/public/css/scss/tag.scss
+++ b/dashboardv2/public/css/scss/tag.scss
@@ -18,8 +18,7 @@
//tag.scss
.tag-tree {
- padding: 0;
- // overflow: auto;
+ padding: 0; // overflow: auto;
li {
list-style: none;
cursor: pointer;
@@ -38,7 +37,7 @@
display: block
}
}
- > ul > li {
+ >ul>li {
line-height: 30px;
margin-left: -20px;
text-indent: 20px;
@@ -93,7 +92,7 @@
cursor: pointer;
}
-.input-group.attributes .form-control:focus + .input-group-addon {
+.input-group.attributes .form-control:focus+.input-group-addon {
border-color: $color_bali_hai_approx;
}
@@ -131,7 +130,7 @@
background-color: $tag_color;
}
}
- > .inputValue {
+ >.inputValue {
padding: 5px;
&:hover {
color: $white;
@@ -345,7 +344,7 @@ form-control .tagInpput {
margin-top: 10px;
}
-.popoverContainer .inputTag > .inputValue {
+.popoverContainer .inputTag>.inputValue {
padding: 2px 5px;
display: inline-block;
width: 100px;
@@ -410,3 +409,32 @@ legend.scheduler-border {
.entityLink {
font-size: 16px;
}
+
+.mainAttrTable {
+ max-height: 180px;
+ overflow: auto;
+ overflow-x: hidden;
+}
+
+table.attriTable {
+ width: 100%;
+ td {
+ border: 1px solid #ddd;
+ padding: 5px 7px;
+ width: 50%;
+ &[data-type="int"],
+ &[data-type="long"],
+ &[data-type="float"],
+ &[data-type="byte"],
+ &[data-type="double"],
+ &[data-type="short"] {
+ text-align: right;
+ }
+ }
+ th {
+ text-align: center !important;
+ border: 1px solid #ddd;
+ padding: 5px 7px;
+ width: 50%;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/71f9c065/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js b/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
index c45755f..56379a4 100644
--- a/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
+++ b/dashboardv2/public/js/views/detail_page/DetailPageLayoutView.js
@@ -104,7 +104,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'collection', 'id', 'entityDefCollection', 'typeHeaders', 'enumDefCollection'));
+ _.extend(this, _.pick(options, 'collection', 'id', 'entityDefCollection', 'typeHeaders', 'enumDefCollection', 'classificationDefCollection'));
this.bindEvents();
},
bindEvents: function() {
@@ -172,7 +172,8 @@ define(['require',
typeHeaders: this.typeHeaders,
entityDefCollection: this.entityDefCollection,
fetchCollection: this.fetchCollection.bind(that),
- enumDefCollection: this.enumDefCollection
+ enumDefCollection: this.enumDefCollection,
+ classificationDefCollection: this.classificationDefCollection
}
this.getEntityDef(obj);
this.renderTagTableLayoutView(obj);
http://git-wip-us.apache.org/repos/asf/atlas/blob/71f9c065/dashboardv2/public/js/views/tag/TagDetailTableLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/tag/TagDetailTableLayoutView.js b/dashboardv2/public/js/views/tag/TagDetailTableLayoutView.js
index 45cdd6b..bac8b4b 100644
--- a/dashboardv2/public/js/views/tag/TagDetailTableLayoutView.js
+++ b/dashboardv2/public/js/views/tag/TagDetailTableLayoutView.js
@@ -64,7 +64,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'entity', 'guid', 'term', 'entityName', 'fetchCollection', 'enumDefCollection'));
+ _.extend(this, _.pick(options, 'entity', 'guid', 'term', 'entityName', 'fetchCollection', 'enumDefCollection', 'classificationDefCollection'));
this.collectionObject = this.entity;
this.tagTermCollection = new VTagList();
var tagorterm = _.toArray(this.collectionObject.classifications),
@@ -133,16 +133,25 @@ define(['require',
sortable: false,
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
- var values = model.get('attributes'),
- tagValue = 'NA';
- if (!_.isEmpty(values)) {
- var stringArr = [];
- tagValue = "";
- _.each(values, function(val, key) {
- var attrName = "<span>" + _.escape(key) + ":" + _.escape(val) + "</span>";
- stringArr.push(attrName);
+ var values = model.get('attributes');
+ var data = that.classificationDefCollection.findWhere({ 'name': model.get('typeName') });
+ var attributeDefs = Utils.getNestedSuperTypeObj({ data: data.toJSON(), collection: that.classificationDefCollection, attrMerge: true });
+ var tagValue = 'NA',
+ dataType;
+ if (!_.isEmpty(attributeDefs)) {
+ var stringValue = "";
+ _.each(_.sortBy(_.map(attributeDefs, function(obj) {
+ obj['sortKey'] = obj.name && _.isString(obj.name) ? obj.name.toLowerCase() : "-";
+ return obj;
+ }), 'sortKey'), function(sortedObj) {
+ var val = _.isNull(values[sortedObj.name]) ? "-" : values[sortedObj.name],
+ key = sortedObj.name;
+ if (sortedObj.typeName === "date") {
+ val = new Date(val)
+ }
+ stringValue += "<tr><td class='html-cell string-cell renderable'>" + _.escape(key) + "</td><td class='html-cell string-cell renderable' data-type=" + sortedObj.typeName + ">" + _.escape(val) + "</td>";
});
- tagValue += stringArr.join(", ");
+ tagValue = "<div class='mainAttrTable'><table class='attriTable'><tr><th class='html-cell string-cell renderable'>Name</th><th class='html-cell string-cell renderable'>Value</th>" + stringValue + "</table></div>";
}
return tagValue;
}
@@ -248,4 +257,4 @@ define(['require',
}
});
return TagDetailTableLayoutView;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/71f9c065/dashboardv2/public/js/views/tag/addTagModalView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/tag/addTagModalView.js b/dashboardv2/public/js/views/tag/addTagModalView.js
index 97d80bd..e240974 100644
--- a/dashboardv2/public/js/views/tag/addTagModalView.js
+++ b/dashboardv2/public/js/views/tag/addTagModalView.js
@@ -272,10 +272,10 @@ define(['require',
var typeName = Utils.getName(obj, 'typeName');
var typeNameValue = that.enumDefCollection.fullCollection.findWhere({ 'name': typeName });
if (typeNameValue) {
- var str = "<option disabled='disabled'" + (!that.tagModel ? 'selected' : '') + ">-- Select " + typeName + " --</option>";
+ var str = '<option value=""' + (!that.tagModel ? 'selected' : '') + '>-- Select ' + typeName + " --</option>";
var enumValue = typeNameValue.get('elementDefs');
_.each(enumValue, function(key, value) {
- str += '<option ' + (that.tagModel && key.value === that.tagModel.attributes[name] ? 'selected' : '') + '>' + key.value + '</option>';
+ str += '<option ' + ((that.tagModel && key.value === that.tagModel.attributes[name]) ? 'selected' : '') + '>' + key.value + '</option>';
})
that.ui.tagAttribute.append('<div class="form-group"><label>' + name + '</label>' + ' (' + typeName + ')' +
'<select class="form-control attributeInputVal attrName" data-key="' + name + '">' + str + '</select></div>');
@@ -286,7 +286,14 @@ define(['require',
});
that.$('input[data-type="date"]').each(function() {
if (!$(this).data('daterangepicker')) {
- var dateObj = { "singleDatePicker": true, "showDropdowns": true };
+ var dateObj = {
+ "singleDatePicker": true,
+ "showDropdowns": true,
+ "timePicker": true,
+ locale: {
+ format: 'MM/DD/YYYY h:mm A'
+ }
+ };
if (that.tagModel) {
var formatDate = Number(this.value);
dateObj["startDate"] = new Date(formatDate);
[25/39] atlas git commit: ATLAS-1901: Tolerate no propogatetags on
Relationshipdef
Posted by ma...@apache.org.
ATLAS-1901: Tolerate no propogatetags on Relationshipdef
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/cda7f56d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/cda7f56d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/cda7f56d
Branch: refs/heads/feature-odf
Commit: cda7f56d5a71970c21de4e2c49771dc85c4a47f0
Parents: 24a106b
Author: David Radley <da...@uk.ibm.com>
Authored: Wed Jul 19 14:59:30 2017 +0100
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 19 09:39:51 2017 -0700
----------------------------------------------------------------------
.../store/graph/v1/AtlasRelationshipDefStoreV1.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/cda7f56d/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipDefStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipDefStoreV1.java
index ddf0af9..bdde58b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipDefStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipDefStoreV1.java
@@ -480,7 +480,12 @@ public class AtlasRelationshipDefStoreV1 extends AtlasAbstractDefStoreV1 impleme
vertex.setProperty(Constants.RELATIONSHIPTYPE_END2_KEY, AtlasType.toJson(relationshipDef.getEndDef2()));
// Update RelationshipCategory
vertex.setProperty(Constants.RELATIONSHIPTYPE_CATEGORY_KEY, relationshipDef.getRelationshipCategory().name());
- vertex.setProperty(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, relationshipDef.getPropagateTags().name());
+
+ if (relationshipDef.getPropagateTags() == null) {
+ vertex.setProperty(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, AtlasRelationshipDef.PropagateTags.NONE.name());
+ } else {
+ vertex.setProperty(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, relationshipDef.getPropagateTags().name());
+ }
}
private AtlasRelationshipDef toRelationshipDef(AtlasVertex vertex) throws AtlasBaseException {
[04/39] atlas git commit: ATLAS-1733: updated atlas_stop.py script to
work in Windows environment
Posted by ma...@apache.org.
ATLAS-1733: updated atlas_stop.py script to work in Windows environment
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/ad398393
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/ad398393
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/ad398393
Branch: refs/heads/feature-odf
Commit: ad398393f712fdbb689a33883182f65f8ef58ac7
Parents: 8fe110c
Author: Graham Wallis <gr...@uk.ibm.com>
Authored: Tue Jul 11 09:26:45 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 11 09:26:45 2017 -0700
----------------------------------------------------------------------
distro/src/bin/atlas_stop.py | 27 ++++++++++++++++++++++-----
1 file changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/ad398393/distro/src/bin/atlas_stop.py
----------------------------------------------------------------------
diff --git a/distro/src/bin/atlas_stop.py b/distro/src/bin/atlas_stop.py
index a25d25a..66edd90 100755
--- a/distro/src/bin/atlas_stop.py
+++ b/distro/src/bin/atlas_stop.py
@@ -15,8 +15,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+
+
+# Signal handling is OS-specific because there is no SIGKILL on Windows.
+
import os
-from signal import SIGTERM, SIGKILL
+
+if os.name == "nt":
+ # Attempting to import SIGKILL on Windows would cause script to fail.
+ from signal import SIGTERM
+else:
+ from signal import SIGTERM, SIGKILL
+
import sys
import traceback
import time
@@ -63,15 +73,22 @@ def main():
# stop hbase
if mc.is_hbase_local(confdir):
mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "stop", None, None, True)
-
+
if mc.exist_pid(pid):
#after 30 seconds kill it
time.sleep(30)
try:
- sys.stderr.write("did not stop gracefully after 30 seconds seconds: killing with SIGKILL\n")
- os.kill(pid, SIGKILL)
+
+ if os.name == "nt":
+ # If running on Windows then timeout termination uses SIGTERM instead of SIGKILL.
+ sys.stderr.write("did not stop gracefully after 30 seconds: killing process using SIGTERM\n")
+ os.kill(pid, SIGTERM)
+ else:
+ sys.stderr.write("did not stop gracefully after 30 seconds: killing process using SIGKILL\n")
+ os.kill(pid, SIGKILL)
+
except:
- pass
+ pass
if __name__ == '__main__':
try:
[24/39] atlas git commit: ATLAS-1948: export fix to correct the
import order
Posted by ma...@apache.org.
ATLAS-1948: export fix to correct the import order
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/24a106b4
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/24a106b4
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/24a106b4
Branch: refs/heads/feature-odf
Commit: 24a106b4e1a1581b266a453cc6c920bcd06d7a9b
Parents: cfb6b84
Author: ashutoshm <am...@hortonworks.com>
Authored: Tue Jul 18 17:08:34 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 19 00:50:14 2017 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/ExportService.java | 22 ++--
.../atlas/repository/impexp/ZipSource.java | 2 +-
.../store/graph/v1/AtlasEntityStoreV1.java | 119 ++++++++++++++-----
.../repository/impexp/ImportServiceTest.java | 24 +++-
repository/src/test/resources/ctas.zip | Bin 0 -> 7674 bytes
5 files changed, 124 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/24a106b4/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 8f45e9f..de48573 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -98,7 +98,6 @@ public class ExportService {
AtlasExportResult.OperationStatus[] statuses = processItems(request, context);
processTypesDef(context);
-
updateSinkWithOperationMetrics(context, statuses, getOperationDuration(startTime));
} catch(Exception ex) {
LOG.error("Operation failed: ", ex);
@@ -113,6 +112,7 @@ public class ExportService {
}
private void updateSinkWithOperationMetrics(ExportContext context, AtlasExportResult.OperationStatus[] statuses, int duration) throws AtlasBaseException {
+ context.result.getData().getEntityCreationOrder().addAll(context.lineageProcessed);
context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
context.sink.setTypesDef(context.result.getData().getTypesDef());
clearContextData(context);
@@ -201,9 +201,10 @@ public class ExportService {
processEntity(guid, context);
}
- if (!context.guidsLineageToProcess.isEmpty()) {
- context.guidsToProcess.addAll(context.guidsLineageToProcess);
- context.guidsLineageToProcess.clear();
+ if (!context.lineageToProcess.isEmpty()) {
+ context.guidsToProcess.addAll(context.lineageToProcess);
+ context.lineageProcessed.addAll(context.lineageToProcess.getList());
+ context.lineageToProcess.clear();
}
}
} catch (AtlasBaseException excp) {
@@ -295,7 +296,9 @@ public class ExportService {
TraversalDirection direction = context.guidDirection.get(guid);
AtlasEntityWithExtInfo entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid);
- context.result.getData().getEntityCreationOrder().add(entityWithExtInfo.getEntity().getGuid());
+ if(!context.lineageProcessed.contains(guid)) {
+ context.result.getData().getEntityCreationOrder().add(entityWithExtInfo.getEntity().getGuid());
+ }
addEntity(entityWithExtInfo, context);
addTypes(entityWithExtInfo.getEntity(), context);
@@ -651,13 +654,18 @@ public class ExportService {
list.clear();
set.clear();
}
+
+ public List<T> getList() {
+ return list;
+ }
}
private class ExportContext {
final Set<String> guidsProcessed = new HashSet<>();
final UniqueList<String> guidsToProcess = new UniqueList<>();
- final UniqueList<String> guidsLineageToProcess = new UniqueList<>();
+ final UniqueList<String> lineageToProcess = new UniqueList<>();
+ final Set<String> lineageProcessed = new HashSet<>();
final Map<String, TraversalDirection> guidDirection = new HashMap<>();
final Set<String> entityTypes = new HashSet<>();
final Set<String> classificationTypes = new HashSet<>();
@@ -719,7 +727,7 @@ public class ExportService {
}
if(isSuperTypeProcess) {
- guidsLineageToProcess.add(guid);
+ lineageToProcess.add(guid);
}
guidDirection.put(guid, direction);
http://git-wip-us.apache.org/repos/asf/atlas/blob/24a106b4/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index edb816f..4c23582 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -196,7 +196,7 @@ public class ZipSource implements EntityImportStream {
AtlasEntity entity = getEntity(guid);
return entity;
} catch (AtlasBaseException e) {
- e.printStackTrace();
+ LOG.error("getByGuid: {} failed!", guid, e);
return null;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/24a106b4/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
index 5ea4ff2..f340330 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -69,7 +69,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
private final DeleteHandlerV1 deleteHandler;
private final AtlasTypeRegistry typeRegistry;
private final AtlasEntityChangeNotifier entityChangeNotifier;
- private final EntityGraphMapper entityGraphMapper;
+ private final EntityGraphMapper entityGraphMapper;
@Inject
public AtlasEntityStoreV1(DeleteHandlerV1 deleteHandler, AtlasTypeRegistry typeRegistry,
@@ -77,7 +77,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
this.deleteHandler = deleteHandler;
this.typeRegistry = typeRegistry;
this.entityChangeNotifier = entityChangeNotifier;
- this.entityGraphMapper = entityGraphMapper;
+ this.entityGraphMapper = entityGraphMapper;
}
@Override
@@ -123,7 +123,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
@Override
@GraphTransaction
public AtlasEntityWithExtInfo getByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes)
- throws AtlasBaseException {
+ throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getByUniqueAttribute({}, {})", entityType.getTypeName(), uniqAttributes);
}
@@ -136,7 +136,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(),
- uniqAttributes.toString());
+ uniqAttributes.toString());
}
if (LOG.isDebugEnabled()) {
@@ -160,29 +160,36 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
EntityMutationResponse ret = new EntityMutationResponse();
ret.setGuidAssignments(new HashMap<String, String>());
- Set<String> processedGuids = new HashSet<>();
- int streamSize = entityStream.size();
- float currentPercent = 0f;
+ Set<String> processedGuids = new HashSet<>();
+ float currentPercent = 0f;
- while (entityStream.hasNext()) {
- AtlasEntityWithExtInfo entityWithExtInfo = entityStream.getNextEntityWithExtInfo();
+ List<String> residualList = new ArrayList<>();
+ EntityImportStreamWithResidualList entityImportStreamWithResidualList = new EntityImportStreamWithResidualList(entityStream, residualList);
+ while (entityImportStreamWithResidualList.hasNext()) {
+ AtlasEntityWithExtInfo entityWithExtInfo = entityImportStreamWithResidualList.getNextEntityWithExtInfo();
AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
- if(entity == null || processedGuids.contains(entity.getGuid())) {
+ if (entity == null || processedGuids.contains(entity.getGuid())) {
continue;
}
AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, entityStream);
+ try {
+ EntityMutationResponse resp = createOrUpdate(oneEntityStream, false, true);
- EntityMutationResponse resp = createOrUpdate(oneEntityStream, false, true);
- currentPercent = updateImportMetrics(entityWithExtInfo, resp, importResult, processedGuids,
- entityStream.getPosition(), streamSize, currentPercent);
+ if (resp.getGuidAssignments() != null) {
+ ret.getGuidAssignments().putAll(resp.getGuidAssignments());
+ }
- if (resp.getGuidAssignments() != null) {
- ret.getGuidAssignments().putAll(resp.getGuidAssignments());
- }
+ currentPercent = updateImportMetrics(entityWithExtInfo, resp, importResult, processedGuids, entityStream.getPosition(),
+ entityImportStreamWithResidualList.getStreamSize(), currentPercent);
- entityStream.onImportComplete(entity.getGuid());
+ entityStream.onImportComplete(entity.getGuid());
+ } catch (AtlasBaseException e) {
+ if (!updateResidualList(e, residualList, entityWithExtInfo.getEntity().getGuid())) {
+ throw e;
+ }
+ }
}
importResult.getProcessedEntities().addAll(processedGuids);
@@ -191,20 +198,28 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return ret;
}
+ private boolean updateResidualList(AtlasBaseException e, List<String> lineageList, String guid) {
+ if (!e.getAtlasErrorCode().getErrorCode().equals(AtlasErrorCode.INVALID_OBJECT_ID.getErrorCode())) {
+ return false;
+ }
+
+ lineageList.add(guid);
+ return true;
+ }
+
private float updateImportMetrics(AtlasEntityWithExtInfo currentEntity,
EntityMutationResponse resp,
AtlasImportResult importResult,
Set<String> processedGuids,
int currentIndex, int streamSize, float currentPercent) {
-
updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult);
updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult);
updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult);
String lastEntityImported = String.format("entity:last-imported:%s:[%s]:(%s)",
- currentEntity.getEntity().getTypeName(),
- currentIndex,
- currentEntity.getEntity().getGuid());
+ currentEntity.getEntity().getTypeName(),
+ currentIndex,
+ currentEntity.getEntity().getGuid());
return updateImportProgress(LOG, currentIndex + 1, streamSize, currentPercent, lastEntityImported);
}
@@ -214,10 +229,10 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
final double tolerance = 0.000001;
final int MAX_PERCENT = 100;
- float percent = (float) ((currentIndex * MAX_PERCENT)/streamSize);
+ float percent = (float) ((currentIndex * MAX_PERCENT) / streamSize);
boolean updateLog = Double.compare(percent, currentPercent) > tolerance;
float updatedPercent = (MAX_PERCENT < streamSize) ? percent :
- ((updateLog) ? ++currentPercent : currentPercent);
+ ((updateLog) ? ++currentPercent : currentPercent);
if (updateLog) {
log.info("bulkImport(): progress: {}% (of {}) - {}", (int) Math.ceil(percent), streamSize, additionalInfo);
@@ -232,7 +247,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
}
for (AtlasEntityHeader h : list) {
- if(processedGuids.contains(h.getGuid())) {
+ if (processedGuids.contains(h.getGuid())) {
continue;
}
@@ -298,7 +313,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
@Override
@GraphTransaction
public EntityMutationResponse updateEntityAttributeByGuid(String guid, String attrName, Object attrValue)
- throws AtlasBaseException {
+ throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> updateEntityAttributeByGuid({}, {}, {})", guid, attrName, attrValue);
}
@@ -490,8 +505,8 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
List<AtlasClassification> updatedClassifications = new ArrayList<>();
for (AtlasClassification newClassification : newClassifications) {
- String classificationName = newClassification.getTypeName();
- AtlasClassification oldClassification = getClassification(guid, classificationName);
+ String classificationName = newClassification.getTypeName();
+ AtlasClassification oldClassification = getClassification(guid, classificationName);
if (oldClassification == null) {
throw new AtlasBaseException(AtlasErrorCode.CLASSIFICATION_NOT_FOUND, classificationName);
@@ -704,7 +719,8 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
/**
* Validate if classification is not already associated with the entities
- * @param guid unique entity id
+ *
+ * @param guid unique entity id
* @param classifications list of classifications to be associated
*/
private void validateEntityAssociations(String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
@@ -715,7 +731,7 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
if (CollectionUtils.isNotEmpty(entityClassifications) && entityClassifications.contains(newClassification)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "entity: " + guid +
- ", already associated with classification: " + newClassification);
+ ", already associated with classification: " + newClassification);
}
}
}
@@ -734,4 +750,43 @@ public class AtlasEntityStoreV1 implements AtlasEntityStore {
return ret;
}
+
+ private static class EntityImportStreamWithResidualList {
+ private final EntityImportStream stream;
+ private final List<String> residualList;
+ private boolean navigateResidualList;
+ private int currentResidualListIndex;
+
+
+ public EntityImportStreamWithResidualList(EntityImportStream stream, List<String> residualList) {
+ this.stream = stream;
+ this.residualList = residualList;
+ this.navigateResidualList = false;
+ this.currentResidualListIndex = 0;
+ }
+
+ public AtlasEntityWithExtInfo getNextEntityWithExtInfo() {
+ if (navigateResidualList == false) {
+ return stream.getNextEntityWithExtInfo();
+ } else {
+ stream.setPositionUsingEntityGuid(residualList.get(currentResidualListIndex++));
+ return stream.getNextEntityWithExtInfo();
+ }
+ }
+
+ public boolean hasNext() {
+ if (!navigateResidualList) {
+ boolean streamHasNext = stream.hasNext();
+ navigateResidualList = (streamHasNext == false);
+ return streamHasNext ? streamHasNext : (currentResidualListIndex < residualList.size());
+ } else {
+ return (currentResidualListIndex < residualList.size());
+ }
+ }
+
+ public int getStreamSize() {
+ return stream.size() + residualList.size();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/24a106b4/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index de8e7ef..404225c 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -133,4 +133,22 @@ public class ImportServiceTest {
assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 8);
}
+
+ @DataProvider(name = "ctas")
+ public static Object[][] getDataFromCtas(ITestContext context) throws IOException {
+ return getZipSource("ctas.zip");
+ }
+
+ @Test(dataProvider = "ctas")
+ public void importCTAS(ZipSource zipSource) throws IOException, AtlasBaseException {
+ loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0030-hive_model.json", typeDefStore, typeRegistry);
+
+ AtlasImportRequest request = getDefaultImportRequest();
+ runImportWithParameters(getImportService(), getDefaultImportRequest(), zipSource);
+ }
+
+ private ImportService getImportService() {
+ return new ImportService(typeDefStore, entityStore, typeRegistry);
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/24a106b4/repository/src/test/resources/ctas.zip
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/ctas.zip b/repository/src/test/resources/ctas.zip
new file mode 100644
index 0000000..a77966c
Binary files /dev/null and b/repository/src/test/resources/ctas.zip differ
[14/39] atlas git commit: ATLAS-1947: AtlasSearchResult to include
referredEntity headers
Posted by ma...@apache.org.
ATLAS-1947: AtlasSearchResult to include referredEntity headers
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/bcec42e3
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/bcec42e3
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/bcec42e3
Branch: refs/heads/feature-odf
Commit: bcec42e3306c9517c1ded5e7ed538c76cfd29c33
Parents: 0d8f9f8
Author: apoorvnaik <ap...@apache.org>
Authored: Thu Jul 13 09:03:06 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 14 16:02:38 2017 -0700
----------------------------------------------------------------------
.../model/discovery/AtlasSearchResult.java | 78 ++-
.../atlas/model/discovery/SearchParameters.java | 34 +-
.../atlas/discovery/AtlasDiscoveryService.java | 2 +-
.../ClassificationSearchProcessor.java | 198 ++++++
.../atlas/discovery/EntityDiscoveryService.java | 138 +++--
.../atlas/discovery/EntitySearchProcessor.java | 203 ++++++
.../discovery/FullTextSearchProcessor.java | 110 ++++
.../org/apache/atlas/discovery/GremlinStep.java | 389 ------------
.../apache/atlas/discovery/SearchContext.java | 126 ++++
.../apache/atlas/discovery/SearchPipeline.java | 611 -------------------
.../apache/atlas/discovery/SearchProcessor.java | 381 ++++++++++++
.../org/apache/atlas/discovery/SolrStep.java | 288 ---------
.../store/graph/v1/EntityGraphRetriever.java | 20 +-
.../org/apache/atlas/util/SearchTracker.java | 16 +-
.../test/java/org/apache/atlas/TestModules.java | 3 -
.../atlas/web/resources/AdminResource.java | 4 +-
.../apache/atlas/web/rest/DiscoveryREST.java | 2 +-
17 files changed, 1216 insertions(+), 1387 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
index 9513dcb..5827440 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java
@@ -31,6 +31,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
@@ -40,14 +41,15 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class AtlasSearchResult implements Serializable {
- private AtlasQueryType queryType;
- private SearchParameters searchParameters;
- private String queryText;
- private String type;
- private String classification;
- private List<AtlasEntityHeader> entities;
- private AttributeSearchResult attributes;
- private List<AtlasFullTextResult> fullTextResult;
+ private AtlasQueryType queryType;
+ private SearchParameters searchParameters;
+ private String queryText;
+ private String type;
+ private String classification;
+ private List<AtlasEntityHeader> entities;
+ private AttributeSearchResult attributes;
+ private List<AtlasFullTextResult> fullTextResult;
+ private Map<String, AtlasEntityHeader> referredEntities;
public AtlasSearchResult() {}
@@ -62,6 +64,7 @@ public class AtlasSearchResult implements Serializable {
setEntities(null);
setAttributes(null);
setFullTextResult(null);
+ setReferredEntities(null);
}
public AtlasSearchResult(SearchParameters searchParameters) {
@@ -73,6 +76,7 @@ public class AtlasSearchResult implements Serializable {
setEntities(null);
setAttributes(null);
setFullTextResult(null);
+ setReferredEntities(null);
}
}
@@ -80,6 +84,14 @@ public class AtlasSearchResult implements Serializable {
public void setQueryType(AtlasQueryType queryType) { this.queryType = queryType; }
+ public SearchParameters getSearchParameters() {
+ return searchParameters;
+ }
+
+ public void setSearchParameters(SearchParameters searchParameters) {
+ this.searchParameters = searchParameters;
+ }
+
public String getQueryText() { return queryText; }
public void setQueryText(String queryText) { this.queryText = queryText; }
@@ -104,6 +116,17 @@ public class AtlasSearchResult implements Serializable {
public void setFullTextResult(List<AtlasFullTextResult> fullTextResult) { this.fullTextResult = fullTextResult; }
+ public Map<String, AtlasEntityHeader> getReferredEntities() {
+ return referredEntities;
+ }
+
+ public void setReferredEntities(Map<String, AtlasEntityHeader> referredEntities) {
+ this.referredEntities = referredEntities;
+ }
+
+ @Override
+ public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult, referredEntities); }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -116,24 +139,8 @@ public class AtlasSearchResult implements Serializable {
Objects.equals(classification, that.classification) &&
Objects.equals(entities, that.entities) &&
Objects.equals(attributes, that.attributes) &&
- Objects.equals(fullTextResult, that.fullTextResult);
- }
-
- @Override
- public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult); }
-
- @Override
- public String toString() {
- return "AtlasSearchResult{" +
- "queryType=" + queryType +
- ", searchParameters='" + searchParameters + '\'' +
- ", queryText='" + queryText + '\'' +
- ", type=" + type +
- ", classification=" + classification +
- ", entities=" + entities +
- ", attributes=" + attributes +
- ", fullTextResult=" + fullTextResult +
- '}';
+ Objects.equals(fullTextResult, that.fullTextResult) &&
+ Objects.equals(referredEntities, that.referredEntities);
}
public void addEntity(AtlasEntityHeader newEntity) {
@@ -163,12 +170,19 @@ public class AtlasSearchResult implements Serializable {
}
}
- public void setSearchParameters(SearchParameters searchParameters) {
- this.searchParameters = searchParameters;
- }
-
- public SearchParameters getSearchParameters() {
- return searchParameters;
+ @Override
+ public String toString() {
+ return "AtlasSearchResult{" +
+ "queryType=" + queryType +
+ ", searchParameters='" + searchParameters + '\'' +
+ ", queryText='" + queryText + '\'' +
+ ", type=" + type +
+ ", classification=" + classification +
+ ", entities=" + entities +
+ ", attributes=" + attributes +
+ ", fullTextResult=" + fullTextResult +
+ ", referredEntities=" + referredEntities +
+ '}';
}
public enum AtlasQueryType { DSL, FULL_TEXT, GREMLIN, BASIC, ATTRIBUTE }
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
index 30855dc..972c11e 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
@@ -207,9 +207,12 @@ public class SearchParameters {
return Objects.hash(query, typeName, classification, excludeDeletedEntities, limit, offset, entityFilters, tagFilters, attributes);
}
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("SearchParameters{");
+ public StringBuilder toString(StringBuilder sb) {
+ if (sb == null) {
+ sb = new StringBuilder();
+ }
+
+ sb.append('{');
sb.append("query='").append(query).append('\'');
sb.append(", typeName='").append(typeName).append('\'');
sb.append(", classification='").append(classification).append('\'');
@@ -220,7 +223,13 @@ public class SearchParameters {
sb.append(", tagFilters=").append(tagFilters);
sb.append(", attributes=").append(attributes);
sb.append('}');
- return sb.toString();
+
+ return sb;
+ }
+
+ @Override
+ public String toString() {
+ return toString(new StringBuilder()).toString();
}
@@ -297,16 +306,25 @@ public class SearchParameters {
return Objects.hash(attributeName, operator, attributeValue, condition, criterion);
}
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("FilterCriteria{");
+ public StringBuilder toString(StringBuilder sb) {
+ if (sb == null) {
+ sb = new StringBuilder();
+ }
+
+ sb.append('{');
sb.append("attributeName='").append(attributeName).append('\'');
sb.append(", operator=").append(operator);
sb.append(", attributeValue='").append(attributeValue).append('\'');
sb.append(", condition=").append(condition);
sb.append(", criterion=").append(criterion);
sb.append('}');
- return sb.toString();
+
+ return sb;
+ }
+
+ @Override
+ public String toString() {
+ return toString(new StringBuilder()).toString();
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index 030a957..764b548 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -64,5 +64,5 @@ public interface AtlasDiscoveryService {
* @return Matching entities
* @throws AtlasBaseException
*/
- AtlasSearchResult searchUsingBasicQuery(SearchParameters searchParameters) throws AtlasBaseException;
+ AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
new file mode 100644
index 0000000..77b2c7c
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+
+public class ClassificationSearchProcessor extends SearchProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
+
+ private final AtlasIndexQuery indexQuery;
+ private final AtlasGraphQuery allGraphQuery;
+ private final AtlasGraphQuery filterGraphQuery;
+
+ public ClassificationSearchProcessor(SearchContext context) {
+ super(context);
+
+ AtlasClassificationType classificationType = context.getClassificationType();
+ FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
+ Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
+ Set<String> solrAttributes = new HashSet<>();
+ Set<String> gremlinAttributes = new HashSet<>();
+ Set<String> allAttributes = new HashSet<>();
+
+
+ processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
+
+ // for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
+ boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
+
+ if (useSolrSearch) {
+ StringBuilder solrQuery = new StringBuilder();
+
+ constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
+
+ String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
+
+ solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
+ solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
+
+ indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
+ } else {
+ indexQuery = null;
+ }
+
+ AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+
+ allGraphQuery = toGremlinFilterQuery(classificationType, filterCriteria, allAttributes, query);
+
+ query = context.getGraph().query().in(Constants.TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes);
+
+ filterGraphQuery = query; // TODO: filer based on tag attributes
+ }
+
+ @Override
+ public List<AtlasVertex> execute() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ClassificationSearchProcessor.execute({})", context);
+ }
+
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ AtlasPerfTracer perf = null;
+
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ClassificationSearchProcessor.execute(" + context + ")");
+ }
+
+ try {
+ int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
+ int limit = context.getSearchParameters().getLimit();
+ int resultIdx = qryOffset;
+ Set<String> processedGuids = new HashSet<>();
+
+ while (ret.size() < limit) {
+ if (context.terminateSearch()) {
+ LOG.warn("query terminated: {}", context.getSearchParameters());
+
+ break;
+ }
+
+ List<AtlasVertex> classificationVertices;
+
+ if (indexQuery != null) {
+ Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
+
+ if (!queryResult.hasNext()) { // no more results from solr - end of search
+ break;
+ }
+
+ classificationVertices = getVerticesFromIndexQueryResult(queryResult);
+ } else {
+ Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
+
+ if (!queryResult.hasNext()) { // no more results - end of search
+ break;
+ }
+
+ classificationVertices = getVertices(queryResult);
+ }
+
+ qryOffset += limit;
+
+ List<AtlasVertex> entityVertices = new ArrayList<>();
+
+ for (AtlasVertex classificationVertex : classificationVertices) {
+ Iterable<AtlasEdge> edges = classificationVertex.getEdges(AtlasEdgeDirection.IN);
+
+ for (AtlasEdge edge : edges) {
+ AtlasVertex entityVertex = edge.getOutVertex();
+ String guid = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
+
+ if (!processedGuids.contains(guid)) {
+ if (!context.getSearchParameters().getExcludeDeletedEntities() || AtlasGraphUtilsV1.getState(entityVertex) == AtlasEntity.Status.ACTIVE) {
+ entityVertices.add(entityVertex);
+ }
+
+ processedGuids.add(guid);
+ }
+ }
+ }
+
+ entityVertices = super.filter(entityVertices);
+
+ for (AtlasVertex entityVertex : entityVertices) {
+ resultIdx++;
+
+ if (resultIdx < context.getSearchParameters().getOffset()) {
+ continue;
+ }
+
+ ret.add(entityVertex);
+
+ if (ret.size() == limit) {
+ break;
+ }
+ }
+ }
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ClassificationSearchProcessor.execute({}): ret.size()={}", context, ret.size());
+ }
+
+ return ret;
+ }
+
+ @Override
+ public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
+ }
+
+ AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
+
+ query.addConditionsFrom(filterGraphQuery);
+
+ List<AtlasVertex> ret = getVertices(query.vertices().iterator());
+
+ ret = super.filter(ret);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ClassificationSearchProcessor.filter({}): ret.size()={}", entityVertices.size(), ret.size());
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 5068fa5..a4538bd 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.exception.AtlasBaseException;
@@ -29,8 +30,8 @@ import org.apache.atlas.model.discovery.AtlasSearchResult.AtlasQueryType;
import org.apache.atlas.model.discovery.AtlasSearchResult.AttributeSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity.Status;
-import org.apache.atlas.AtlasException;
import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.query.Expressions.AliasExpression;
import org.apache.atlas.query.Expressions.Expression;
import org.apache.atlas.query.Expressions.SelectExpression;
@@ -42,16 +43,16 @@ import org.apache.atlas.query.QueryProcessor;
import org.apache.atlas.query.SelectExpressionHelper;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery.Result;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.*;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
-import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.commons.collections.CollectionUtils;
@@ -67,13 +68,7 @@ import scala.util.parsing.combinator.Parsers.NoSuccess;
import javax.inject.Inject;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
@@ -88,21 +83,20 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final EntityGraphRetriever entityRetriever;
private final AtlasGremlinQueryProvider gremlinQueryProvider;
private final AtlasTypeRegistry typeRegistry;
- private final SearchPipeline searchPipeline;
+ private final GraphBackedSearchIndexer indexer;
private final int maxResultSetSize;
private final int maxTypesCountInIdxQuery;
private final int maxTagsCountInIdxQuery;
@Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
- AtlasGraph graph, SearchPipeline searchPipeline) throws AtlasException {
+ AtlasGraph graph, GraphBackedSearchIndexer indexer) throws AtlasException {
this.graph = graph;
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.indexer = indexer;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry;
- this.searchPipeline = searchPipeline;
-
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
@@ -404,20 +398,85 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
- public AtlasSearchResult searchUsingBasicQuery(SearchParameters searchParameters) throws AtlasBaseException {
+ public AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
- List<AtlasVertex> resultList = searchPipeline.run(searchParameters);
+ SearchContext context = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys());
+
+ List<AtlasVertex> resultList = context.getSearchProcessor().execute();
+
+ // By default any attribute that shows up in the search parameter should be sent back in the response
+ // If additional values are requested then the entityAttributes will be a superset of the all search attributes
+ // and the explicitly requested attribute(s)
+ Set<String> resultAttributes = new HashSet<>();
+ Set<String> entityAttributes = new HashSet<>();
+
+ if (CollectionUtils.isNotEmpty(searchParameters.getAttributes())) {
+ resultAttributes.addAll(searchParameters.getAttributes());
+ }
+
+ for (String resultAttribute : resultAttributes) {
+ AtlasAttribute attribute = context.getEntityType().getAttribute(resultAttribute);
+
+ if (attribute != null) {
+ AtlasType attributeType = attribute.getAttributeType();
+
+ if (attributeType instanceof AtlasArrayType) {
+ attributeType = ((AtlasArrayType) attributeType).getElementType();
+ }
+
+ if (attributeType instanceof AtlasEntityType || attributeType instanceof AtlasObjectIdType) {
+ entityAttributes.add(resultAttribute);
+ }
+ }
+ }
for (AtlasVertex atlasVertex : resultList) {
- AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, searchParameters.getAttributes());
+ AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, resultAttributes);
ret.addEntity(entity);
+
+ // populate ret.referredEntities
+ for (String entityAttribute : entityAttributes) {
+ Object attrValue = entity.getAttribute(entityAttribute);
+
+ if (attrValue instanceof AtlasObjectId) {
+ AtlasObjectId objId = (AtlasObjectId)attrValue;
+
+ if (ret.getReferredEntities() == null) {
+ ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
+ }
+
+ if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
+ ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ }
+ } else if (attrValue instanceof Collection) {
+ Collection objIds = (Collection)attrValue;
+
+ for (Object obj : objIds) {
+ if (obj instanceof AtlasObjectId) {
+ AtlasObjectId objId = (AtlasObjectId)obj;
+
+ if (ret.getReferredEntities() == null) {
+ ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
+ }
+
+ if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
+ ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ }
+ }
+ }
+ }
+ }
}
return ret;
}
+ public int getMaxResultSetSize() {
+ return maxResultSetSize;
+ }
+
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
String classficationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery);
@@ -447,28 +506,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText.toString());
}
- private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesCountInIdxQuery) {
- AtlasClassificationType classification = typeRegistry.getClassificationTypeByName(classificationName);
- Set<String> typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes() : null;
-
- if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
- return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
- }
-
- return "";
- }
-
- private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesCountInIdxQuery) {
- AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
- Set<String> typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes() : null;
-
- if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
- return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
- }
-
- return "";
- }
-
private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params, boolean excludeDeletedEntities) throws AtlasBaseException {
List<AtlasFullTextResult> ret = new ArrayList<>();
Iterator<Result> iter = query.vertices();
@@ -570,8 +607,25 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
}
- public int getMaxResultSetSize() {
- return maxResultSetSize;
+ private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesCountInIdxQuery) {
+ AtlasClassificationType classification = typeRegistry.getClassificationTypeByName(classificationName);
+ Set<String> typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes() : null;
+
+ if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
+ return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+ }
+
+ return "";
}
+ private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesCountInIdxQuery) {
+ AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
+ Set<String> typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes() : null;
+
+ if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
+ return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+ }
+
+ return "";
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
new file mode 100644
index 0000000..605cb15
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class EntitySearchProcessor extends SearchProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor");
+
+ private final AtlasIndexQuery indexQuery;
+ private final AtlasGraphQuery partialGraphQuery;
+ private final AtlasGraphQuery allGraphQuery;
+
+ public EntitySearchProcessor(SearchContext context) {
+ super(context);
+
+ AtlasEntityType entityType = context.getEntityType();
+ AtlasClassificationType classificationType = context.getClassificationType();
+ FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ Set<String> solrAttributes = new HashSet<>();
+ Set<String> gremlinAttributes = new HashSet<>();
+ Set<String> allAttributes = new HashSet<>();
+
+
+ processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
+
+ boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY && canApplySolrFilter(entityType, filterCriteria, false);
+
+ if (useSolrSearch) {
+ StringBuilder solrQuery = new StringBuilder();
+
+ constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes);
+
+ String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
+
+ solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
+ solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
+
+ indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
+
+ if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null) {
+ AtlasGraphQuery query = context.getGraph().query();
+
+ addClassificationNameConditionIfNecessary(query);
+
+ partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
+ } else {
+ partialGraphQuery = null;
+ }
+ } else {
+ indexQuery = null;
+ partialGraphQuery = null;
+ }
+
+ AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+
+ addClassificationNameConditionIfNecessary(query);
+
+ allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
+
+ if (context.getSearchParameters().getExcludeDeletedEntities()) {
+ allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
+ }
+ }
+
+ @Override
+ public List<AtlasVertex> execute() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> EntitySearchProcessor.execute({})", context);
+ }
+
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ AtlasPerfTracer perf = null;
+
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntitySearchProcessor.execute(" + context + ")");
+ }
+
+ try {
+ int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
+ int limit = context.getSearchParameters().getLimit();
+ int resultIdx = qryOffset;
+
+ while (ret.size() < limit) {
+ if (context.terminateSearch()) {
+ LOG.warn("query terminated: {}", context.getSearchParameters());
+
+ break;
+ }
+
+ List<AtlasVertex> vertices;
+
+ if (indexQuery != null) {
+ Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
+
+ if (!queryResult.hasNext()) { // no more results from solr - end of search
+ break;
+ }
+
+ vertices = getVerticesFromIndexQueryResult(queryResult);
+
+ if (partialGraphQuery != null) {
+ AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices));
+
+ guidQuery.addConditionsFrom(partialGraphQuery);
+
+ vertices = getVertices(guidQuery.vertices().iterator());
+ }
+ } else {
+ Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
+
+ if (!queryResult.hasNext()) { // no more results from query - end of search
+ break;
+ }
+
+ vertices = getVertices(queryResult);
+ }
+
+ qryOffset += limit;
+
+ vertices = super.filter(vertices);
+
+ for (AtlasVertex vertex : vertices) {
+ resultIdx++;
+
+ if (resultIdx < context.getSearchParameters().getOffset()) {
+ continue;
+ }
+
+ ret.add(vertex);
+
+ if (ret.size() == limit) {
+ break;
+ }
+ }
+ }
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== EntitySearchProcessor.execute({}): ret.size()={}", context, ret.size());
+ }
+
+ return ret;
+ }
+
+ @Override
+ public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> EntitySearchProcessor.filter({})", entityVertices.size());
+ }
+
+ AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
+
+ query.addConditionsFrom(allGraphQuery);
+
+ List<AtlasVertex> ret = getVertices(query.vertices().iterator());
+
+ ret = super.filter(ret);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== EntitySearchProcessor.filter({}): ret.size()={}", entityVertices.size(), ret.size());
+ }
+
+ return ret;
+ }
+
+ private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) {
+ if (context.getClassificationType() != null && !context.needClassificationProcessor()) {
+ query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
new file mode 100644
index 0000000..4ddd642
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.utils.AtlasPerfTracer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+
+public class FullTextSearchProcessor extends SearchProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FullTextSearchProcessor");
+
+ private final AtlasIndexQuery indexQuery;
+
+ public FullTextSearchProcessor(SearchContext context) {
+ super(context);
+
+ SearchParameters searchParameters = context.getSearchParameters();
+ String queryString = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
+
+ indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString);
+ }
+
+ @Override
+ public List<AtlasVertex> execute() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> FullTextSearchProcessor.execute({})", context);
+ }
+
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ AtlasPerfTracer perf = null;
+
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessor.execute(" + context + ")");
+ }
+
+ try {
+ int qryOffset = nextProcessor == null ? context.getSearchParameters().getOffset() : 0;
+ int limit = context.getSearchParameters().getLimit();
+ int resultIdx = qryOffset;
+
+ while (ret.size() < limit) {
+ if (context.terminateSearch()) {
+ LOG.warn("query terminated: {}", context.getSearchParameters());
+
+ break;
+ }
+
+ Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
+
+ if (!idxQueryResult.hasNext()) { // no more results from solr - end of search
+ break;
+ }
+
+ qryOffset += limit;
+
+ List<AtlasVertex> vertices = getVerticesFromIndexQueryResult(idxQueryResult);
+
+ vertices = super.filter(vertices);
+
+ for (AtlasVertex vertex : vertices) {
+ resultIdx++;
+
+ if (resultIdx < context.getSearchParameters().getOffset()) {
+ continue;
+ }
+
+ ret.add(vertex);
+
+ if (ret.size() == limit) {
+ break;
+ }
+ }
+ }
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
+ }
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/GremlinStep.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/GremlinStep.java b/repository/src/main/java/org/apache/atlas/discovery/GremlinStep.java
deleted file mode 100644
index 1056b3e..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/GremlinStep.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
-import org.apache.atlas.model.discovery.SearchParameters.Operator;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasStructType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.utils.AtlasPerfTracer;
-import org.apache.commons.collections.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.atlas.discovery.SearchPipeline.IndexResultType;
-import static org.apache.atlas.discovery.SearchPipeline.PipelineContext;
-import static org.apache.atlas.discovery.SearchPipeline.PipelineStep;
-import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator;
-import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.MatchingOperator;
-
-@Component
-public class GremlinStep implements PipelineStep {
- private static final Logger LOG = LoggerFactory.getLogger(GremlinStep.class);
- private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("GremlinSearchStep");
-
- private final AtlasGraph graph;
- private final AtlasTypeRegistry typeRegistry;
-
- enum GremlinFilterQueryType { TAG, ENTITY }
-
- @Inject
- public GremlinStep(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
- this.graph = graph;
- this.typeRegistry = typeRegistry;
- }
-
- @Override
- public void execute(PipelineContext context) throws AtlasBaseException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> GremlinStep.execute({})", context);
- }
-
- if (context == null) {
- throw new AtlasBaseException("Can't start search without any context");
- }
-
- AtlasPerfTracer perf = null;
-
- if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GremlinSearchStep.execute(" + context + ")");
- }
-
- final Iterator<AtlasVertex> result;
-
- if (context.hasIndexResults()) {
- // We have some results from the indexed step, let's proceed accordingly
- if (context.getIndexResultType() == IndexResultType.TAG) {
- // Index search was done on tag and filters
- if (context.isTagProcessingComplete()) {
- LOG.debug("GremlinStep.execute(): index has completely processed tag, further TAG filtering not needed");
-
- Set<String> taggedVertexGUIDs = new HashSet<>();
-
- Iterator<AtlasIndexQuery.Result> tagVertexIterator = context.getIndexResultsIterator();
-
- while (tagVertexIterator.hasNext()) {
- // Find out which Vertex has this outgoing edge
- AtlasVertex vertex = tagVertexIterator.next().getVertex();
- Iterable<AtlasEdge> edges = vertex.getEdges(AtlasEdgeDirection.IN);
-
- for (AtlasEdge edge : edges) {
- String guid = AtlasGraphUtilsV1.getIdFromVertex(edge.getOutVertex());
-
- taggedVertexGUIDs.add(guid);
- }
- }
-
- // No entities are tagged (actually this check is already done)
- if (!taggedVertexGUIDs.isEmpty()) {
- result = processEntity(taggedVertexGUIDs, context);
- } else {
- result = null;
- }
- } else {
- result = processTagAndEntity(Collections.<String>emptySet(), context);
- }
- } else if (context.getIndexResultType() == IndexResultType.TEXT) {
- // Index step processed full-text;
- Set<String> entityIDs = getVertexIDs(context.getIndexResultsIterator());
-
- result = processTagAndEntity(entityIDs, context);
- } else if (context.getIndexResultType() == IndexResultType.ENTITY) {
- // Index step processed entity and it's filters; tag filter wouldn't be set
- Set<String> entityIDs = getVertexIDs(context.getIndexResultsIterator());
-
- result = processEntity(entityIDs, context);
- } else {
- result = null;
- }
- } else {
- // No index results, need full processing in Gremlin
- if (context.getClassificationType() != null) {
- // Process tag and filters first, then entity filters
- result = processTagAndEntity(Collections.<String>emptySet(), context);
- } else {
- result = processEntity(Collections.<String>emptySet(), context);
- }
- }
-
- context.setGremlinResultIterator(result);
-
- AtlasPerfTracer.log(perf);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== GremlinStep.execute({})", context);
- }
- }
-
- private Iterator<AtlasVertex> processEntity(Set<String> entityGUIDs, PipelineContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> GremlinStep.processEntity(entityGUIDs={})", entityGUIDs);
- }
-
- final Iterator<AtlasVertex> ret;
-
- SearchParameters searchParameters = context.getSearchParameters();
- AtlasEntityType entityType = context.getEntityType();
-
- if (entityType != null) {
- AtlasGraphQuery entityFilterQuery = context.getGraphQuery("ENTITY_FILTER");
-
- if (entityFilterQuery == null) {
- entityFilterQuery = graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, entityType.getTypeAndAllSubTypes());
-
- if (searchParameters.getEntityFilters() != null) {
- toGremlinFilterQuery(GremlinFilterQueryType.ENTITY, entityType, searchParameters.getEntityFilters(), entityFilterQuery, context);
- }
-
- if (searchParameters.getExcludeDeletedEntities()) {
- entityFilterQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
- }
-
- context.cacheGraphQuery("ENTITY_FILTER", entityFilterQuery);
- }
-
- // Now get all vertices
- if (CollectionUtils.isEmpty(entityGUIDs)) {
- ret = entityFilterQuery.vertices(context.getCurrentOffset(), context.getMaxLimit()).iterator();
- } else {
- AtlasGraphQuery guidQuery = graph.query().in(Constants.GUID_PROPERTY_KEY, entityGUIDs);
-
- if (entityFilterQuery != null) {
- guidQuery.addConditionsFrom(entityFilterQuery);
- } else if (searchParameters.getExcludeDeletedEntities()) {
- guidQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
- }
-
- ret = guidQuery.vertices(context.getMaxLimit()).iterator();
- }
- } else if (CollectionUtils.isNotEmpty(entityGUIDs)) {
- AtlasGraphQuery guidQuery = graph.query().in(Constants.GUID_PROPERTY_KEY, entityGUIDs);
-
- if (searchParameters.getExcludeDeletedEntities()) {
- guidQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
- }
-
- Iterable<AtlasVertex> vertices = guidQuery.vertices(context.getMaxLimit());
-
- ret = vertices.iterator();
- } else {
- ret = null;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== GremlinStep.processEntity(entityGUIDs={})", entityGUIDs);
- }
-
- return ret;
- }
-
- private Iterator<AtlasVertex> processTagAndEntity(Set<String> entityGUIDs, PipelineContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> GremlinStep.processTagAndEntity(entityGUIDs={})", entityGUIDs);
- }
-
- final Iterator<AtlasVertex> ret;
-
- AtlasClassificationType classificationType = context.getClassificationType();
-
- if (classificationType != null) {
- AtlasGraphQuery tagVertexQuery = context.getGraphQuery("TAG_VERTEX");
-
- if (tagVertexQuery == null) {
- tagVertexQuery = graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
-
- SearchParameters searchParameters = context.getSearchParameters();
-
- // Do tag filtering first as it'll return a smaller subset of vertices
- if (searchParameters.getTagFilters() != null) {
- toGremlinFilterQuery(GremlinFilterQueryType.TAG, classificationType, searchParameters.getTagFilters(), tagVertexQuery, context);
- }
-
- context.cacheGraphQuery("TAG_VERTEX", tagVertexQuery);
- }
-
- if (tagVertexQuery != null) {
- Set<String> taggedVertexGuids = new HashSet<>();
- // Now get all vertices after adjusting offset for each iteration
- LOG.debug("Firing TAG query");
-
- Iterator<AtlasVertex> tagVertexIterator = tagVertexQuery.vertices(context.getCurrentOffset(), context.getMaxLimit()).iterator();
-
- while (tagVertexIterator.hasNext()) {
- // Find out which Vertex has this outgoing edge
- Iterable<AtlasEdge> edges = tagVertexIterator.next().getEdges(AtlasEdgeDirection.IN);
- for (AtlasEdge edge : edges) {
- String guid = AtlasGraphUtilsV1.getIdFromVertex(edge.getOutVertex());
- taggedVertexGuids.add(guid);
- }
- }
-
- entityGUIDs = taggedVertexGuids;
- }
- }
-
- if (!entityGUIDs.isEmpty()) {
- ret = processEntity(entityGUIDs, context);
- } else {
- ret = null;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== GremlinStep.processTagAndEntity(entityGUIDs={})", entityGUIDs);
- }
-
- return ret;
- }
-
- private Set<String> getVertexIDs(Iterator<AtlasIndexQuery.Result> idxResultsIterator) {
- Set<String> guids = new HashSet<>();
- while (idxResultsIterator.hasNext()) {
- AtlasVertex vertex = idxResultsIterator.next().getVertex();
- String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
- guids.add(guid);
- }
- return guids;
- }
-
- private Set<String> getVertexIDs(Iterable<AtlasVertex> vertices) {
- Set<String> guids = new HashSet<>();
- for (AtlasVertex vertex : vertices) {
- String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
- guids.add(guid);
- }
- return guids;
- }
-
- private AtlasGraphQuery toGremlinFilterQuery(GremlinFilterQueryType queryType, AtlasStructType type, FilterCriteria criteria,
- AtlasGraphQuery query, PipelineContext context) {
- if (criteria.getCondition() != null) {
- if (criteria.getCondition() == Condition.AND) {
- for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGremlinFilterQuery(queryType, type, filterCriteria, graph.query(), context);
- query.addConditionsFrom(nestedQuery);
- }
- } else {
- List<AtlasGraphQuery> orConditions = new LinkedList<>();
-
- for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGremlinFilterQuery(queryType, type, filterCriteria, graph.query(), context);
- // FIXME: Something might not be right here as the queries are getting overwritten sometimes
- orConditions.add(graph.query().createChildQuery().addConditionsFrom(nestedQuery));
- }
-
- if (!orConditions.isEmpty()) {
- query.or(orConditions);
- }
- }
- } else {
- String attrName = criteria.getAttributeName();
- String attrValue = criteria.getAttributeValue();
- Operator operator = criteria.getOperator();
-
- try {
- // If attribute belongs to supertype then adjust the name accordingly
- final String qualifiedAttributeName;
- final boolean attrProcessed;
-
- if (queryType == GremlinFilterQueryType.TAG) {
- qualifiedAttributeName = type.getQualifiedAttributeName(attrName);
- attrProcessed = context.hasProcessedTagAttribute(qualifiedAttributeName);
- } else {
- qualifiedAttributeName = type.getQualifiedAttributeName(attrName);
- attrProcessed = context.hasProcessedEntityAttribute(qualifiedAttributeName);
- }
-
- // Check if the qualifiedAttribute has been processed
- if (!attrProcessed) {
- switch (operator) {
- case LT:
- query.has(qualifiedAttributeName, ComparisionOperator.LESS_THAN, attrValue);
- break;
- case LTE:
- query.has(qualifiedAttributeName, ComparisionOperator.LESS_THAN_EQUAL, attrValue);
- break;
- case GT:
- query.has(qualifiedAttributeName, ComparisionOperator.GREATER_THAN, attrValue);
- break;
- case GTE:
- query.has(qualifiedAttributeName, ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
- break;
- case EQ:
- query.has(qualifiedAttributeName, ComparisionOperator.EQUAL, attrValue);
- break;
- case NEQ:
- query.has(qualifiedAttributeName, ComparisionOperator.NOT_EQUAL, attrValue);
- break;
- case LIKE:
- // TODO: Maybe we need to validate pattern
- query.has(qualifiedAttributeName, MatchingOperator.REGEX, getLikeRegex(attrValue));
- break;
- case CONTAINS:
- query.has(qualifiedAttributeName, MatchingOperator.REGEX, getContainsRegex(attrValue));
- break;
- case STARTS_WITH:
- query.has(qualifiedAttributeName, MatchingOperator.PREFIX, attrValue);
- break;
- case ENDS_WITH:
- query.has(qualifiedAttributeName, MatchingOperator.REGEX, getSuffixRegex(attrValue));
- break;
- case IN:
- LOG.warn("{}: unsupported operator. Ignored", operator);
- break;
- }
- }
- } catch (AtlasBaseException e) {
- LOG.error("toGremlinFilterQuery(): failed for attrName=" + attrName + "; operator=" + operator + "; attrValue=" + attrValue, e);
- }
- }
-
- return query;
- }
-
- private String getContainsRegex(String attributeValue) {
- return ".*" + attributeValue + ".*";
- }
-
- private String getSuffixRegex(String attributeValue) {
- return ".*" + attributeValue;
- }
-
- private String getLikeRegex(String attributeValue) { return ".*" + attributeValue + ".*"; }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
new file mode 100644
index 0000000..2125d61
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.Set;
+
+
+public class SearchContext {
+ private final SearchParameters searchParameters;
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasGraph graph;
+ private final Set<String> indexedKeys;
+ private final AtlasEntityType entityType;
+ private final AtlasClassificationType classificationType;
+ private SearchProcessor searchProcessor;
+ private boolean terminateSearch = false;
+
+ public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) {
+ this.searchParameters = searchParameters;
+ this.typeRegistry = typeRegistry;
+ this.graph = graph;
+ this.indexedKeys = indexedKeys;
+ this.entityType = typeRegistry.getEntityTypeByName(searchParameters.getTypeName());
+ this.classificationType = typeRegistry.getClassificationTypeByName(searchParameters.getClassification());
+
+ if (needFullTextrocessor()) {
+ addProcessor(new FullTextSearchProcessor(this));
+ }
+
+ if (needClassificationProcessor()) {
+ addProcessor(new ClassificationSearchProcessor(this));
+ }
+
+ if (needEntityProcessor()) {
+ addProcessor(new EntitySearchProcessor(this));
+
+ }
+ }
+
+ public SearchParameters getSearchParameters() { return searchParameters; }
+
+ public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; }
+
+ public AtlasGraph getGraph() { return graph; }
+
+ public Set<String> getIndexedKeys() { return indexedKeys; }
+
+ public AtlasEntityType getEntityType() { return entityType; }
+
+ public AtlasClassificationType getClassificationType() { return classificationType; }
+
+ public SearchProcessor getSearchProcessor() { return searchProcessor; }
+
+ public boolean terminateSearch() { return this.terminateSearch; }
+
+ public void terminateSearch(boolean terminateSearch) { this.terminateSearch = terminateSearch; }
+
+ public StringBuilder toString(StringBuilder sb) {
+ if (sb == null) {
+ sb = new StringBuilder();
+ }
+
+ sb.append("searchParameters=");
+
+ if (searchParameters != null) {
+ searchParameters.toString(sb);
+ }
+
+ return sb;
+ }
+
+ @Override
+ public String toString() {
+ return toString(new StringBuilder()).toString();
+ }
+
+ public boolean needFullTextrocessor() {
+ return StringUtils.isNotEmpty(searchParameters.getQuery());
+ }
+
+ public boolean needClassificationProcessor() {
+ return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters()) || entityType == null);
+ }
+
+ public boolean needEntityProcessor() {
+ return entityType != null;
+ }
+
+ private boolean hasAttributeFilter(FilterCriteria filterCriteria) {
+ return filterCriteria != null &&
+ (CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
+ }
+
+ private void addProcessor(SearchProcessor processor) {
+ if (this.searchProcessor == null) {
+ this.searchProcessor = processor;
+ } else {
+ this.searchProcessor.addProcessor(processor);
+ }
+ }
+}
[18/39] atlas git commit: ATLAS-1919: Refactored ZipSink to record
committed guids
Posted by ma...@apache.org.
ATLAS-1919: Refactored ZipSink to record committed guids
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/3962057c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/3962057c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/3962057c
Branch: refs/heads/feature-odf
Commit: 3962057c640e1fbecb4cfb6a785475e441a7de1d
Parents: bcabde9
Author: ashutoshm <am...@hortonworks.com>
Authored: Mon Jul 10 08:25:58 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jul 17 10:12:18 2017 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/ExportService.java | 4 +
.../apache/atlas/repository/impexp/ZipSink.java | 21 ++++++
.../atlas/repository/impexp/ZipSinkTest.java | 79 +++++++++++++++++---
3 files changed, 93 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/3962057c/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 3538cfd..8f45e9f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -433,6 +433,10 @@ public class ExportService {
}
private void addEntity(AtlasEntityWithExtInfo entity, ExportContext context) throws AtlasBaseException {
+ if(context.sink.hasEntity(entity.getEntity().getGuid())) {
+ return;
+ }
+
context.sink.add(entity);
context.result.incrementMeticsCounter(String.format("entity:%s", entity.getEntity().getTypeName()));
http://git-wip-us.apache.org/repos/asf/atlas/blob/3962057c/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSink.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSink.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSink.java
index 4bb04da..17ebbf1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSink.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSink.java
@@ -27,7 +27,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -35,6 +37,8 @@ public class ZipSink {
private static final Logger LOG = LoggerFactory.getLogger(ZipSink.class);
private ZipOutputStream zipOutputStream;
+ final Set<String> guids = new HashSet<>();
+
public ZipSink(OutputStream outputStream) {
zipOutputStream = new ZipOutputStream(outputStream);
@@ -43,11 +47,13 @@ public class ZipSink {
public void add(AtlasEntity entity) throws AtlasBaseException {
String jsonData = convertToJSON(entity);
saveToZip(entity.getGuid(), jsonData);
+ recordAddedEntityGuids(entity);
}
public void add(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException {
String jsonData = convertToJSON(entityWithExtInfo);
saveToZip(entityWithExtInfo.getEntity().getGuid(), jsonData);
+ recordAddedEntityGuids(entityWithExtInfo);
}
public void setResult(AtlasExportResult result) throws AtlasBaseException {
@@ -100,4 +106,19 @@ public class ZipSink {
zipOutputStream.write(payload.getBytes());
zipOutputStream.closeEntry();
}
+
+ public boolean hasEntity(String guid) {
+ return guids.contains(guid);
+ }
+
+ private void recordAddedEntityGuids(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ guids.add(entityWithExtInfo.getEntity().getGuid());
+ if(entityWithExtInfo.getReferredEntities() != null) {
+ guids.addAll(entityWithExtInfo.getReferredEntities().keySet());
+ }
+ }
+
+ private void recordAddedEntityGuids(AtlasEntity entity) {
+ guids.add(entity.getGuid());
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3962057c/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
index 635caf7..e8bbeb5 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
@@ -21,6 +21,7 @@ package org.apache.atlas.repository.impexp;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.type.AtlasType;
import org.testng.Assert;
@@ -35,11 +36,15 @@ import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import static org.testng.Assert.*;
+
public class ZipSinkTest {
private ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
private ZipSink zipSink;
private List<String> defaultExportOrder = new ArrayList<>(Arrays.asList("a", "b", "c", "d"));
private AtlasExportResult defaultExportResult;
+ private String knownEntityGuidFormat = "111-222-333-%s";
+
private void initZipSinkWithExportOrder() throws AtlasBaseException {
zipSink = new ZipSink(byteArrayOutputStream);
@@ -80,7 +85,7 @@ public class ZipSinkTest {
@Test
public void correctInit_succeeds() throws AtlasBaseException {
initZipSinkWithExportOrder();
- Assert.assertTrue(true);
+ assertTrue(true);
Assert.assertNotNull(zipSink);
}
@@ -95,11 +100,11 @@ public class ZipSinkTest {
Assert.assertNull(zis.getNextEntry());
} catch (IOException e) {
- Assert.assertTrue(false);
+ assertTrue(false);
}
} catch (AtlasBaseException e) {
- Assert.assertTrue(false, "No exception should be thrown.");
+ assertTrue(false, "No exception should be thrown.");
}
}
@@ -109,7 +114,7 @@ public class ZipSinkTest {
ZipInputStream zis = getZipInputStreamForDefaultExportOrder();
ZipEntry ze = zis.getNextEntry();
- Assert.assertEquals(ze.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString());
+ assertEquals(ze.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString());
}
@Test
@@ -118,28 +123,80 @@ public class ZipSinkTest {
ZipInputStream zis = getZipInputStreamForDefaultExportOrder();
zis.getNextEntry();
- Assert.assertEquals(getZipEntryAsStream(zis).replace("\"", "'"), "['a','b','c','d']");
+ assertEquals(getZipEntryAsStream(zis).replace("\"", "'"), "['a','b','c','d']");
}
@Test
public void zipWithExactlyTwoEntries_ContentsVerified() throws AtlasBaseException, IOException {
ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
- useZipSinkToCreateZipWithTwoEntries(byteOutputStream);
+ useZipSinkToCreateEntries(byteOutputStream);
ByteArrayInputStream bis = new ByteArrayInputStream(byteOutputStream.toByteArray());
ZipInputStream zipStream = new ZipInputStream(bis);
ZipEntry entry = zipStream.getNextEntry();
- Assert.assertEquals(getZipEntryAsStream(zipStream), "[\"a\",\"b\",\"c\",\"d\"]");
- Assert.assertEquals(entry.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString());
+ assertEquals(getZipEntryAsStream(zipStream), "[\"a\",\"b\",\"c\",\"d\"]");
+ assertEquals(entry.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString());
entry = zipStream.getNextEntry();
- Assert.assertEquals(entry.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_INFO_NAME.toString());
- Assert.assertTrue(compareJsonWithObject(getZipEntryAsStream(zipStream), defaultExportResult));
+ assertEquals(entry.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_INFO_NAME.toString());
+ assertTrue(compareJsonWithObject(getZipEntryAsStream(zipStream), defaultExportResult));
+ }
+
+ @Test
+ public void recordsEntityEntries() throws AtlasBaseException {
+ ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ ZipSink zs = new ZipSink(byteOutputStream);
+
+ AtlasEntity entity = new AtlasEntity();
+ entity.setGuid(String.format(knownEntityGuidFormat, 0));
+
+ zs.add(entity);
+ assertTrue(zs.hasEntity(String.format(knownEntityGuidFormat, 0)));
+
+ zs.close();
+ }
+
+ @Test
+ public void recordsEntityWithExtInfoEntries() throws AtlasBaseException {
+ final int max_entries = 3;
+ ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ ZipSink zs = new ZipSink(byteOutputStream);
+
+ AtlasEntity entity = new AtlasEntity();
+ entity.setGuid(String.format(knownEntityGuidFormat, 0));
+
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo(entity);
+ addReferredEntities(entityWithExtInfo, max_entries);
+
+ zs.add(entityWithExtInfo);
+ for (int i = 0; i <= max_entries; i++) {
+ String g = String.format(knownEntityGuidFormat, i);
+ assertTrue(zs.hasEntity(g));
+ }
+
+ zs.close();
+ }
+
+ private void addReferredEntities(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, int maxEntries) {
+
+ for (int i = 1; i <= maxEntries; i++) {
+ AtlasEntity entity1 = new AtlasEntity();
+ entity1.setGuid(String.format(knownEntityGuidFormat, i));
+ entityWithExtInfo.addReferredEntity(entity1);
+ }
+ }
+
+ @Test
+ public void recordsDoesNotRecordEntityEntries() throws AtlasBaseException {
+ initZipSinkWithExportOrder();
+
+ assertNotNull(zipSink);
+ assertFalse(zipSink.hasEntity(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()));
}
- private void useZipSinkToCreateZipWithTwoEntries(ByteArrayOutputStream byteOutputStream) throws AtlasBaseException {
+ private void useZipSinkToCreateEntries(ByteArrayOutputStream byteOutputStream) throws AtlasBaseException {
ZipSink zs = new ZipSink(byteOutputStream);
zs.setExportOrder(defaultExportOrder);
zs.setResult(getDefaultExportResult());
[38/39] atlas git commit: ATLAS-1981: fix for unit test failures
Posted by ma...@apache.org.
ATLAS-1981: fix for unit test failures
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7fb09b0f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7fb09b0f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7fb09b0f
Branch: refs/heads/feature-odf
Commit: 7fb09b0f964f4b4373c885b427359ccf58dfff26
Parents: de25191
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Sun Jul 23 20:46:32 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jul 24 00:48:25 2017 -0700
----------------------------------------------------------------------
.../org/apache/atlas/type/AtlasStructType.java | 17 +++++++++++++----
.../atlas/services/EntityDiscoveryServiceTest.java | 12 ++++++------
2 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/7fb09b0f/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 049a537..9e263d7 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -34,11 +34,12 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/**
* class that implements behaviour of a struct-type.
@@ -714,13 +715,21 @@ public class AtlasStructType extends AtlasType {
return key;
}
- public static String escapeIndexQueryValue(Set<String> values) {
+ public static String escapeIndexQueryValue(Collection<String> values) {
StringBuilder sb = new StringBuilder();
sb.append(BRACE_OPEN_CHAR);
- for (String value : values) {
- sb.append(escapeIndexQueryValue(value)).append(SPACE_CHAR);
+
+ if (CollectionUtils.isNotEmpty(values)) {
+ Iterator<String> iter = values.iterator();
+
+ sb.append(escapeIndexQueryValue(iter.next()));
+
+ while (iter.hasNext()) {
+ sb.append(SPACE_CHAR).append(escapeIndexQueryValue(iter.next()));
+ }
}
+
sb.append(BRACE_CLOSE_CHAR);
return sb.toString();
http://git-wip-us.apache.org/repos/asf/atlas/blob/7fb09b0f/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
index dfb2ee2..5a0739c 100644
--- a/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/services/EntityDiscoveryServiceTest.java
@@ -50,7 +50,7 @@ public class EntityDiscoveryServiceTest {
AtlasEntityDef typeTest3 = null;
AtlasEntityDef typeWithSubTypes = null;
- private final int maxTypesCountInIdxQuery = 10;
+ private final int maxTypesStrLengthInIdxQuery = 55;
@Inject
EntityDiscoveryService discoveryService;
@@ -81,17 +81,17 @@ public class EntityDiscoveryServiceTest {
@Test
public void getSubTypesForType_NullStringReturnsEmptyString() throws Exception {
- invokeGetSubTypesForType(null, maxTypesCountInIdxQuery);
+ invokeGetSubTypesForType(null, maxTypesStrLengthInIdxQuery);
}
@Test
public void getSubTypesForType_BlankStringReturnsEmptyString() throws Exception {
- invokeGetSubTypesForType(" ", maxTypesCountInIdxQuery);
+ invokeGetSubTypesForType(" ", maxTypesStrLengthInIdxQuery);
}
@Test
public void getSubTypesForType_EmptyStringReturnsEmptyString() throws Exception {
- invokeGetSubTypesForType("", maxTypesCountInIdxQuery);
+ invokeGetSubTypesForType("", maxTypesStrLengthInIdxQuery);
}
@Test
@@ -103,7 +103,7 @@ public class EntityDiscoveryServiceTest {
@Test
public void getSubTypeForTypeWithSubTypes_ReturnsOrClause() throws Exception {
- String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, maxTypesCountInIdxQuery);
+ String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, maxTypesStrLengthInIdxQuery);
assertTrue(s.startsWith("("));
assertTrue(s.contains(TEST_TYPE_WITH_SUB_TYPES));
@@ -115,7 +115,7 @@ public class EntityDiscoveryServiceTest {
@Test
public void getSubTypeForTypeWithSubTypes_ReturnsEmptyString() throws Exception {
- String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, 2);
+ String s = invokeGetSubTypesForType(TEST_TYPE_WITH_SUB_TYPES, 20);
assertTrue(StringUtils.isBlank(s));
}
[19/39] atlas git commit: ATLAS-1925: basic search fixes for
ATLAS-1917, ATLAS-1922, ATLAS-1926
Posted by ma...@apache.org.
ATLAS-1925: basic search fixes for ATLAS-1917, ATLAS-1922, ATLAS-1926
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/abc4856e
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/abc4856e
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/abc4856e
Branch: refs/heads/feature-odf
Commit: abc4856e94e24837eee0dc5efc1763a5ceba6fc9
Parents: 3962057
Author: apoorvnaik <ap...@apache.org>
Authored: Mon Jul 17 14:09:22 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jul 17 17:07:53 2017 -0700
----------------------------------------------------------------------
.../atlas/discovery/EntityDiscoveryService.java | 106 ++++++++++---------
.../apache/atlas/discovery/SearchProcessor.java | 36 +++++--
2 files changed, 82 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/abc4856e/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index a4538bd..c7a624f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -55,6 +55,7 @@ import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
+import org.apache.atlas.util.SearchTracker;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
@@ -84,17 +85,19 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final AtlasGremlinQueryProvider gremlinQueryProvider;
private final AtlasTypeRegistry typeRegistry;
private final GraphBackedSearchIndexer indexer;
+ private final SearchTracker searchTracker;
private final int maxResultSetSize;
private final int maxTypesCountInIdxQuery;
private final int maxTagsCountInIdxQuery;
@Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
- AtlasGraph graph, GraphBackedSearchIndexer indexer) throws AtlasException {
+ AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker) throws AtlasException {
this.graph = graph;
this.graphPersistenceStrategy = new DefaultGraphPersistenceStrategy(metadataRepository);
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.indexer = indexer;
+ this.searchTracker = searchTracker;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry;
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
@@ -401,73 +404,78 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
public AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
- SearchContext context = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys());
+ SearchContext context = new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys());
+ String searchID = searchTracker.add(context); // For future cancellations
- List<AtlasVertex> resultList = context.getSearchProcessor().execute();
+ try {
+ List<AtlasVertex> resultList = context.getSearchProcessor().execute();
- // By default any attribute that shows up in the search parameter should be sent back in the response
- // If additional values are requested then the entityAttributes will be a superset of the all search attributes
- // and the explicitly requested attribute(s)
- Set<String> resultAttributes = new HashSet<>();
- Set<String> entityAttributes = new HashSet<>();
+ // By default any attribute that shows up in the search parameter should be sent back in the response
+ // If additional values are requested then the entityAttributes will be a superset of the all search attributes
+ // and the explicitly requested attribute(s)
+ Set<String> resultAttributes = new HashSet<>();
+ Set<String> entityAttributes = new HashSet<>();
- if (CollectionUtils.isNotEmpty(searchParameters.getAttributes())) {
- resultAttributes.addAll(searchParameters.getAttributes());
- }
+ if (CollectionUtils.isNotEmpty(searchParameters.getAttributes())) {
+ resultAttributes.addAll(searchParameters.getAttributes());
+ }
- for (String resultAttribute : resultAttributes) {
- AtlasAttribute attribute = context.getEntityType().getAttribute(resultAttribute);
+ for (String resultAttribute : resultAttributes) {
+ AtlasAttribute attribute = context.getEntityType().getAttribute(resultAttribute);
- if (attribute != null) {
- AtlasType attributeType = attribute.getAttributeType();
+ if (attribute != null) {
+ AtlasType attributeType = attribute.getAttributeType();
- if (attributeType instanceof AtlasArrayType) {
- attributeType = ((AtlasArrayType) attributeType).getElementType();
- }
+ if (attributeType instanceof AtlasArrayType) {
+ attributeType = ((AtlasArrayType) attributeType).getElementType();
+ }
- if (attributeType instanceof AtlasEntityType || attributeType instanceof AtlasObjectIdType) {
- entityAttributes.add(resultAttribute);
+ if (attributeType instanceof AtlasEntityType || attributeType instanceof AtlasObjectIdType) {
+ entityAttributes.add(resultAttribute);
+ }
}
}
- }
- for (AtlasVertex atlasVertex : resultList) {
- AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, resultAttributes);
+ for (AtlasVertex atlasVertex : resultList) {
+ AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, resultAttributes);
- ret.addEntity(entity);
+ ret.addEntity(entity);
- // populate ret.referredEntities
- for (String entityAttribute : entityAttributes) {
- Object attrValue = entity.getAttribute(entityAttribute);
+ // populate ret.referredEntities
+ for (String entityAttribute : entityAttributes) {
+ Object attrValue = entity.getAttribute(entityAttribute);
- if (attrValue instanceof AtlasObjectId) {
- AtlasObjectId objId = (AtlasObjectId)attrValue;
+ if (attrValue instanceof AtlasObjectId) {
+ AtlasObjectId objId = (AtlasObjectId) attrValue;
- if (ret.getReferredEntities() == null) {
- ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
- }
+ if (ret.getReferredEntities() == null) {
+ ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
+ }
- if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
- ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
- }
- } else if (attrValue instanceof Collection) {
- Collection objIds = (Collection)attrValue;
+ if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
+ ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ }
+ } else if (attrValue instanceof Collection) {
+ Collection objIds = (Collection) attrValue;
- for (Object obj : objIds) {
- if (obj instanceof AtlasObjectId) {
- AtlasObjectId objId = (AtlasObjectId)obj;
+ for (Object obj : objIds) {
+ if (obj instanceof AtlasObjectId) {
+ AtlasObjectId objId = (AtlasObjectId) obj;
- if (ret.getReferredEntities() == null) {
- ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
- }
+ if (ret.getReferredEntities() == null) {
+ ret.setReferredEntities(new HashMap<String, AtlasEntityHeader>());
+ }
- if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
- ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
+ ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ }
}
}
}
}
}
+ } finally {
+ searchTracker.remove(searchID);
}
return ret;
@@ -478,8 +486,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
- String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
- String classficationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery);
+ String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
+ String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery);
StringBuilder queryText = new StringBuilder();
@@ -495,12 +503,12 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
queryText.append(typeFilter);
}
- if (! StringUtils.isEmpty(classficationFilter)) {
+ if (! StringUtils.isEmpty(classificationFilter)) {
if (queryText.length() > 0) {
queryText.append(" AND ");
}
- queryText.append(classficationFilter);
+ queryText.append(classificationFilter);
}
return String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText.toString());
http://git-wip-us.apache.org/repos/asf/atlas/blob/abc4856e/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 1a2d997..29430ef 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -52,6 +52,7 @@ public abstract class SearchProcessor {
public static final String BRACE_CLOSE_STR = " )";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
+ private static final char[] OFFENDING_CHARS = {'@', '/', ' '}; // This can grow as we discover corner cases
static
{
@@ -60,9 +61,9 @@ public abstract class SearchProcessor {
OPERATOR_MAP.put(SearchParameters.Operator.LTE,"v.\"%s\": [* TO %s]");
OPERATOR_MAP.put(SearchParameters.Operator.GTE,"v.\"%s\": [%s TO *]");
OPERATOR_MAP.put(SearchParameters.Operator.EQ,"v.\"%s\": %s");
- OPERATOR_MAP.put(SearchParameters.Operator.NEQ,"v.\"%s\": (NOT %s)");
- OPERATOR_MAP.put(SearchParameters.Operator.IN, "v.\"%s\": (%s)");
- OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "v.\"%s\": (%s)");
+ OPERATOR_MAP.put(SearchParameters.Operator.NEQ,"-" + "v.\"%s\": %s");
+ OPERATOR_MAP.put(SearchParameters.Operator.IN, "v.\"%s\": (%s)"); // this should be a list of quoted strings
+ OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "v.\"%s\": (%s)"); // this should be regex pattern
OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, "v.\"%s\": (%s*)");
OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, "v.\"%s\": (*%s)");
OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, "v.\"%s\": (*%s*)");
@@ -184,7 +185,7 @@ public abstract class SearchProcessor {
if (filterCriteria != null) {
LOG.debug("Processing Filters");
- String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes);
+ String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) {
solrQuery.append(AND_STR).append(filterQuery);
@@ -196,27 +197,31 @@ public abstract class SearchProcessor {
}
}
- private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes) {
- return toSolrQuery(type, criteria, solrAttributes, new StringBuilder());
+ private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, int level) {
+ return toSolrQuery(type, criteria, solrAttributes, new StringBuilder(), level);
}
- private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, StringBuilder sb) {
+ private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, StringBuilder sb, int level) {
if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- String nestedQuery = toSolrQuery(type, filterCriteria, solrAttributes);
+ String nestedQuery = toSolrQuery(type, filterCriteria, solrAttributes, level + 1);
if (StringUtils.isNotEmpty(nestedQuery)) {
if (nestedExpression.length() > 0) {
nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING);
}
-
+ // todo: when a neq operation is nested and occurs in the beginning of the query, solr has issues
nestedExpression.append(nestedQuery);
}
}
- return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression.toString()).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
+ if (level == 0) {
+ return nestedExpression.length() > 0 ? sb.append(nestedExpression).toString() : EMPTY_STRING;
+ } else {
+ return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
+ }
} else if (solrAttributes.contains(criteria.getAttributeName())){
return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
} else {
@@ -231,7 +236,12 @@ public abstract class SearchProcessor {
String qualifiedName = type.getQualifiedAttributeName(attrName);
if (OPERATOR_MAP.get(op) != null) {
- ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal);
+ if (hasOffendingChars(attrVal)) {
+ // FIXME: if attrVal has offending chars & op is contains, endsWith, startsWith, solr doesn't like it and results are skewed
+ ret = String.format(OPERATOR_MAP.get(op), qualifiedName, "\"" + attrVal + "\"");
+ } else {
+ ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal);
+ }
}
} catch (AtlasBaseException ex) {
LOG.warn(ex.getMessage());
@@ -378,4 +388,8 @@ public abstract class SearchProcessor {
return defaultValue;
}
+
+ private boolean hasOffendingChars(String str) {
+ return StringUtils.containsAny(str, OFFENDING_CHARS);
+ }
}
[09/39] atlas git commit: ATLAS-1944: Implemented ShutdownableThread
for HookConsumer
Posted by ma...@apache.org.
ATLAS-1944: Implemented ShutdownableThread for HookConsumer
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/18745cf4
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/18745cf4
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/18745cf4
Branch: refs/heads/feature-odf
Commit: 18745cf4b98af9c45e853daa280342dde8da1300
Parents: b0470f5
Author: ashutoshm <am...@hortonworks.com>
Authored: Wed Jul 12 14:43:45 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jul 13 13:25:15 2017 -0700
----------------------------------------------------------------------
.../notification/NotificationHookConsumer.java | 56 +++++++++++---------
1 file changed, 31 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/18745cf4/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
index 9e5b864..0dea0e2 100644
--- a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
+++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
@@ -19,6 +19,7 @@ package org.apache.atlas.notification;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import kafka.utils.ShutdownableThread;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.AtlasServiceException;
@@ -28,7 +29,11 @@ import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.kafka.AtlasKafkaMessage;
import org.apache.atlas.listener.ActiveStateChangeHandler;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.notification.hook.HookNotification.EntityCreateRequest;
+import org.apache.atlas.notification.hook.HookNotification.EntityDeleteRequest;
import org.apache.atlas.notification.hook.HookNotification.EntityPartialUpdateRequest;
+import org.apache.atlas.notification.hook.HookNotification.EntityUpdateRequest;
+import org.apache.atlas.notification.hook.HookNotification.HookNotificationMessage;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
@@ -41,11 +46,12 @@ import org.apache.atlas.web.filters.AuditFilter;
import org.apache.atlas.web.service.ServiceState;
import org.apache.atlas.web.util.DateTimeHelper;
import org.apache.commons.configuration.Configuration;
+import org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
-import org.apache.kafka.common.TopicPartition;
+
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
@@ -56,14 +62,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import static org.apache.atlas.AtlasClientV2.CREATE_ENTITY;
-import static org.apache.atlas.AtlasClientV2.DELETE_ENTITY_BY_ATTRIBUTE;
-import static org.apache.atlas.AtlasClientV2.UPDATE_ENTITY;
-import static org.apache.atlas.AtlasClientV2.UPDATE_ENTITY_BY_ATTRIBUTE;
-import static org.apache.atlas.notification.hook.HookNotification.EntityCreateRequest;
-import static org.apache.atlas.notification.hook.HookNotification.EntityDeleteRequest;
-import static org.apache.atlas.notification.hook.HookNotification.EntityUpdateRequest;
-import static org.apache.atlas.notification.hook.HookNotification.HookNotificationMessage;
+import static org.apache.atlas.AtlasClientV2.*;
/**
* Consumer of notifications from hooks e.g., hive hook etc.
@@ -80,7 +79,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
public static final String CONSUMER_THREADS_PROPERTY = "atlas.notification.hook.numthreads";
public static final String CONSUMER_RETRIES_PROPERTY = "atlas.notification.hook.maxretries";
public static final String CONSUMER_FAILEDCACHESIZE_PROPERTY = "atlas.notification.hook.failedcachesize";
- public static final String CONSUMER_RETRY_INTERVAL="atlas.notification.consumer.retry.interval";
+ public static final String CONSUMER_RETRY_INTERVAL = "atlas.notification.consumer.retry.interval";
public static final int SERVER_READY_WAIT_TIME_MS = 1000;
private final AtlasEntityStore atlasEntityStore;
@@ -177,7 +176,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
/**
* Start Kafka consumer threads that read from Kafka topic when server is activated.
- *
+ * <p>
* Since the consumers create / update entities to the shared backend store, only the active instance
* should perform this activity. Hence, these threads are started only on server activation.
*/
@@ -189,7 +188,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
/**
* Stop Kafka consumer threads that read from Kafka topic when server is de-activated.
- *
+ * <p>
* Since the consumers create / update entities to the shared backend store, only the active instance
* should perform this activity. Hence, these threads are stopped only on server deactivation.
*/
@@ -205,18 +204,18 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
}
}
- class HookConsumer implements Runnable {
+ class HookConsumer extends ShutdownableThread {
private final NotificationConsumer<HookNotificationMessage> consumer;
private final AtomicBoolean shouldRun = new AtomicBoolean(false);
private List<HookNotificationMessage> failedMessages = new ArrayList<>();
public HookConsumer(NotificationConsumer<HookNotificationMessage> consumer) {
+ super("atlas-hook-consumer-thread", false);
this.consumer = consumer;
}
-
@Override
- public void run() {
+ public void doWork() {
shouldRun.set(true);
if (!serverAvailable(new NotificationHookConsumer.Timer())) {
@@ -226,7 +225,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
while (shouldRun.get()) {
try {
List<AtlasKafkaMessage<HookNotificationMessage>> messages = consumer.receive(1000L);
- for (AtlasKafkaMessage<HookNotificationMessage> msg : messages){
+ for (AtlasKafkaMessage<HookNotificationMessage> msg : messages) {
handleMessage(msg);
}
} catch (Throwable t) {
@@ -267,15 +266,17 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
if (numRetries == 0) { // audit only on the first attempt
audit(messageUser, UPDATE_ENTITY_BY_ATTRIBUTE.getMethod(),
- String.format(UPDATE_ENTITY_BY_ATTRIBUTE.getPath(), partialUpdateRequest.getTypeName()));
+ String.format(UPDATE_ENTITY_BY_ATTRIBUTE.getPath(), partialUpdateRequest.getTypeName()));
}
Referenceable referenceable = partialUpdateRequest.getEntity();
entities = instanceConverter.toAtlasEntity(referenceable);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(partialUpdateRequest.getTypeName());
- String guid = AtlasGraphUtilsV1.getGuidByUniqueAttributes(entityType, new HashMap<String, Object>(){
- { put(partialUpdateRequest.getAttribute(), partialUpdateRequest.getAttributeValue()); }
+ String guid = AtlasGraphUtilsV1.getGuidByUniqueAttributes(entityType, new HashMap<String, Object>() {
+ {
+ put(partialUpdateRequest.getAttribute(), partialUpdateRequest.getAttributeValue());
+ }
});
// There should only be one root entity
@@ -289,13 +290,15 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
if (numRetries == 0) { // audit only on the first attempt
audit(messageUser, DELETE_ENTITY_BY_ATTRIBUTE.getMethod(),
- String.format(DELETE_ENTITY_BY_ATTRIBUTE.getPath(), deleteRequest.getTypeName()));
+ String.format(DELETE_ENTITY_BY_ATTRIBUTE.getPath(), deleteRequest.getTypeName()));
}
try {
AtlasEntityType type = (AtlasEntityType) typeRegistry.getType(deleteRequest.getTypeName());
atlasEntityStore.deleteByUniqueAttributes(type,
- new HashMap<String, Object>() {{ put(deleteRequest.getAttribute(), deleteRequest.getAttributeValue()); }});
+ new HashMap<String, Object>() {{
+ put(deleteRequest.getAttribute(), deleteRequest.getAttributeValue());
+ }});
} catch (ClassCastException cle) {
LOG.error("Failed to do a partial update on Entity");
}
@@ -319,10 +322,10 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
break;
} catch (Throwable e) {
LOG.warn("Error handling message", e);
- try{
+ try {
LOG.info("Sleeping for {} ms before retry", consumerRetryInterval);
Thread.sleep(consumerRetryInterval);
- }catch (InterruptedException ie){
+ } catch (InterruptedException ie) {
LOG.error("Notification consumer thread sleep interrupted");
}
@@ -379,9 +382,12 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
return true;
}
- public void stop() {
+ @Override
+ public void shutdown() {
+ super.initiateShutdown();
shouldRun.set(false);
consumer.close();
+ super.awaitShutdown();
}
}
@@ -393,4 +399,4 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
AuditFilter.audit(messageUser, THREADNAME_PREFIX, method, LOCALHOST, path, LOCALHOST,
DateTimeHelper.formatDateUTC(new Date()));
}
-}
+}
\ No newline at end of file
[17/39] atlas git commit: ATLAS-1908: updated NotificationConsumer
with a method removed earlier to avoid breaking existing usage (like in
Ranger)
Posted by ma...@apache.org.
ATLAS-1908: updated NotificationConsumer with a method removed earlier to avoid breaking existing usage (like in Ranger)
Change-Id: Ib8a7f338da7fd0f710fc683da87871e3d9c32035
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/bcabde9b
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/bcabde9b
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/bcabde9b
Branch: refs/heads/feature-odf
Commit: bcabde9bbe8361bc7b4461b395dd2ffcb0906962
Parents: eddab3b
Author: nixonrodrigues <ni...@apache.org>
Authored: Sun Jul 16 00:14:29 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sat Jul 15 14:50:04 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/kafka/AtlasKafkaConsumer.java | 9 ++++++++-
.../org/apache/atlas/notification/NotificationConsumer.java | 9 +++++++++
.../notification/AbstractNotificationConsumerTest.java | 5 +++++
3 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcabde9b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
index 52d0916..d431176 100644
--- a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
+++ b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
@@ -52,9 +52,15 @@ public class AtlasKafkaConsumer<T> extends AbstractNotificationConsumer<T> {
}
public List<AtlasKafkaMessage<T>> receive() {
+ return this.receive(this.pollTimeoutMilliSeconds);
+ }
+
+ @Override
+ public List<AtlasKafkaMessage<T>> receive(long timeoutMilliSeconds) {
+
List<AtlasKafkaMessage<T>> messages = new ArrayList();
- ConsumerRecords<?, ?> records = kafkaConsumer.poll(pollTimeoutMilliSeconds);
+ ConsumerRecords<?, ?> records = kafkaConsumer.poll(timeoutMilliSeconds);
if (records != null) {
for (ConsumerRecord<?, ?> record : records) {
@@ -70,6 +76,7 @@ public class AtlasKafkaConsumer<T> extends AbstractNotificationConsumer<T> {
}
return messages;
+
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcabde9b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
index 6d1c08a..0bd75e1 100644
--- a/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
+++ b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
@@ -44,4 +44,13 @@ public interface NotificationConsumer<T> {
* @return List containing kafka message and partionId and offset.
*/
List<AtlasKafkaMessage<T>> receive();
+
+ /**
+ * Fetch data for the topics from Kafka
+ * @param timeoutMilliSeconds poll timeout
+ * @return List containing kafka message and partionId and offset.
+ */
+ List<AtlasKafkaMessage<T>> receive(long timeoutMilliSeconds);
+
+
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcabde9b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
index 68fe3d7..bcee00c 100644
--- a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
+++ b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
@@ -204,6 +204,11 @@ public class AbstractNotificationConsumerTest {
@Override
public List<AtlasKafkaMessage<T>> receive() {
+ return receive(1000L);
+ }
+
+ @Override
+ public List<AtlasKafkaMessage<T>> receive(long timeoutMilliSeconds) {
List<AtlasKafkaMessage<T>> tempMessageList = new ArrayList();
for(Object json : messageList) {
tempMessageList.add(new AtlasKafkaMessage(deserializer.deserialize((String)json), -1, -1));
[31/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering (# 4)
Posted by ma...@apache.org.
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 4)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/e0fb7dc1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/e0fb7dc1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/e0fb7dc1
Branch: refs/heads/feature-odf
Commit: e0fb7dc17000724fada89cb573db9b4fa1654a2e
Parents: 3d3be40
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Thu Jul 20 01:19:57 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jul 20 14:23:18 2017 -0700
----------------------------------------------------------------------
.../ClassificationSearchProcessor.java | 35 +++++++++++++-------
.../atlas/discovery/EntitySearchProcessor.java | 26 ++++++++-------
.../discovery/FullTextSearchProcessor.java | 27 ++++++++++++---
.../apache/atlas/discovery/SearchProcessor.java | 18 +++++-----
4 files changed, 68 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0fb7dc1/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index b6e0de5..745f9d7 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -58,7 +58,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
- constructTypeTestQuery(solrQuery, classificationType, typeAndSubTypes);
+ constructTypeTestQuery(solrQuery, typeAndSubTypes);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
@@ -95,16 +95,22 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
try {
- final int startIdx = context.getSearchParameters().getOffset();
- final int limit = context.getSearchParameters().getLimit();
- int qryOffset = nextProcessor == null ? startIdx : 0;
- int resultIdx = qryOffset;
+ final int startIdx = context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
+
+ // query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
+ // have been dropped: like non-active-entities or duplicate-entities (same entity pointed to by multiple
+ // classifications in the result)
+ //
+ // first 'startIdx' number of entries will be ignored
+ int qryOffset = 0;
+ int resultIdx = qryOffset;
final Set<String> processedGuids = new HashSet<>();
final List<AtlasVertex> entityVertices = new ArrayList<>();
final List<AtlasVertex> classificationVertices = new ArrayList<>();
-
for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear();
classificationVertices.clear();
@@ -138,15 +144,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
for (AtlasEdge edge : edges) {
AtlasVertex entityVertex = edge.getOutVertex();
- String guid = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
- if (!processedGuids.contains(guid)) {
- if (!context.getSearchParameters().getExcludeDeletedEntities() || AtlasGraphUtilsV1.getState(entityVertex) == AtlasEntity.Status.ACTIVE) {
- entityVertices.add(entityVertex);
- }
+ if (activeOnly && AtlasGraphUtilsV1.getState(entityVertex) != AtlasEntity.Status.ACTIVE) {
+ continue;
+ }
+
+ String guid = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
- processedGuids.add(guid);
+ if (processedGuids.contains(guid)) {
+ continue;
}
+
+ entityVertices.add(entityVertex);
+
+ processedGuids.add(guid);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0fb7dc1/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 6f629eb..a3525c9 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -18,6 +18,7 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
@@ -60,7 +61,7 @@ public class EntitySearchProcessor extends SearchProcessor {
StringBuilder solrQuery = new StringBuilder();
if (typeSearchBySolr) {
- constructTypeTestQuery(solrQuery, entityType, typeAndSubTypes);
+ constructTypeTestQuery(solrQuery, typeAndSubTypes);
}
if (attrSearchBySolr) {
@@ -70,6 +71,10 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (solrQuery.length() > 0) {
+ if (context.getSearchParameters().getExcludeDeletedEntities()) {
+ constructStateTestQuery(solrQuery);
+ }
+
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
@@ -128,10 +133,14 @@ public class EntitySearchProcessor extends SearchProcessor {
}
try {
- final int startIdx = context.getSearchParameters().getOffset();
- final int limit = context.getSearchParameters().getLimit();
- int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery == null)) ? startIdx : 0;
- int resultIdx = qryOffset;
+ final int startIdx = context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+
+ // when subsequent filtering stages are involved, query should start at 0 even though startIdx can be higher
+ //
+ // first 'startIdx' number of entries will be ignored
+ int qryOffset = (nextProcessor != null || (graphQuery != null && indexQuery != null)) ? 0 : startIdx;
+ int resultIdx = qryOffset;
final List<AtlasVertex> entityVertices = new ArrayList<>();
@@ -154,13 +163,6 @@ public class EntitySearchProcessor extends SearchProcessor {
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
- // skip non-entity vertices
- if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) {
- LOG.warn("EntitySearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId()); // might cause duplicate entries in result
-
- continue;
- }
-
entityVertices.add(vertex);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0fb7dc1/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 22d91e0..1b19a0e 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -18,6 +18,7 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
@@ -74,6 +75,10 @@ public class FullTextSearchProcessor extends SearchProcessor {
}
}
+ if (context.getSearchParameters().getExcludeDeletedEntities()) {
+ queryString.append(AND_STR).append("(ACTIVE)");
+ }
+
queryString.append(")");
indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString());
@@ -94,10 +99,16 @@ public class FullTextSearchProcessor extends SearchProcessor {
}
try {
- final int startIdx = context.getSearchParameters().getOffset();
- final int limit = context.getSearchParameters().getLimit();
- int qryOffset = nextProcessor == null ? startIdx : 0;
- int resultIdx = qryOffset;
+ final int startIdx = context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
+
+ // query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
+ // have been dropped: like vertices of non-entity or non-active-entity
+ //
+ // first 'startIdx' number of entries will be ignored
+ int qryOffset = 0;
+ int resultIdx = qryOffset;
final List<AtlasVertex> entityVertices = new ArrayList<>();
@@ -121,8 +132,14 @@ public class FullTextSearchProcessor extends SearchProcessor {
// skip non-entity vertices
if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) {
- LOG.warn("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId()); // might cause duplicate entries in result
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
+ }
+
+ continue;
+ }
+ if (activeOnly && AtlasGraphUtilsV1.getState(vertex) != AtlasEntity.Status.ACTIVE) {
continue;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0fb7dc1/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 2e75dfe..7950127 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
return ret;
}
- protected void constructTypeTestQuery(StringBuilder solrQuery, AtlasStructType type, Set<String> typeAndAllSubTypes) {
+ protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
@@ -193,14 +193,6 @@ public abstract class SearchProcessor {
.append(typeAndSubtypesString)
.append(")");
}
-
- if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
- if (solrQuery.length() > 0) {
- solrQuery.append(AND_STR);
- }
-
- solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
- }
}
protected void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> solrAttributes) {
@@ -219,6 +211,14 @@ public abstract class SearchProcessor {
}
}
+ protected void constructStateTestQuery(StringBuilder solrQuery) {
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
+ }
+
private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, int level) {
return toSolrQuery(type, criteria, solrAttributes, new StringBuilder(), level);
}
[26/39] atlas git commit: ATLAS-1914: clear button in search UI to
clear filter attributes
Posted by ma...@apache.org.
ATLAS-1914: clear button in search UI to clear filter attributes
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/5527afb0
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/5527afb0
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/5527afb0
Branch: refs/heads/feature-odf
Commit: 5527afb0d4cdf4414f2e95e0d419392dc78c9d9a
Parents: cda7f56
Author: kevalbhatt <kb...@apache.org>
Authored: Wed Jul 19 19:36:01 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 19 10:02:26 2017 -0700
----------------------------------------------------------------------
dashboardv2/public/js/router/Router.js | 31 +++++----
.../views/business_catalog/SideNavLayoutView.js | 4 +-
.../public/js/views/search/QueryBuilderView.js | 18 ++++-
.../public/js/views/search/SearchLayoutView.js | 69 ++++++++++++--------
.../public/js/views/tag/TagLayoutView.js | 13 +++-
5 files changed, 88 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/5527afb0/dashboardv2/public/js/router/Router.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/router/Router.js b/dashboardv2/public/js/router/Router.js
index 8a1d241..f705843 100644
--- a/dashboardv2/public/js/router/Router.js
+++ b/dashboardv2/public/js/router/Router.js
@@ -53,6 +53,10 @@ define([
'enumDefCollection': this.enumDefCollection,
'classificationDefCollection': this.classificationDefCollection
}
+ this.filterObj = {
+ 'tagFilters': JSON.parse(Utils.localStorage.getValue('tagFilters')),
+ 'entityFilters': JSON.parse(Utils.localStorage.getValue('entityFilters'))
+ }
},
bindCommonEvents: function() {
var that = this;
@@ -125,6 +129,7 @@ define([
App.rSideNav.show(new SideNavLayoutView(
_.extend({
'url': url,
+ 'filterObj': that.filterObj
}, that.preFetchedCollectionLists)
));
} else {
@@ -156,7 +161,7 @@ define([
App.rNHeader.show(new Header());
if (!App.rSideNav.currentView) {
App.rSideNav.show(new SideNavLayoutView(
- _.extend({}, that.preFetchedCollectionLists)
+ _.extend({ 'filterObj': that.filterObj }, that.preFetchedCollectionLists)
));
} else {
App.rSideNav.currentView.selectTab();
@@ -193,7 +198,8 @@ define([
}
App.rSideNav.show(new SideNavLayoutView(
_.extend({
- 'tag': tagName
+ 'tag': tagName,
+ 'filterObj': that.filterObj
}, that.preFetchedCollectionLists)
));
} else {
@@ -231,17 +237,13 @@ define([
'views/business_catalog/SideNavLayoutView',
'views/search/SearchDetailLayoutView',
], function(Header, BusinessCatalogLayoutView, SideNavLayoutView, SearchDetailLayoutView) {
- var paramObj = Utils.getUrlState.getQueryParams(),
- filterObj = {
- 'tagFilters': JSON.parse(Utils.localStorage.getValue('tagFilters')),
- 'entityFilters': JSON.parse(Utils.localStorage.getValue('entityFilters'))
- }
+ var paramObj = Utils.getUrlState.getQueryParams();
App.rNHeader.show(new Header());
if (!App.rSideNav.currentView) {
App.rSideNav.show(new SideNavLayoutView(
_.extend({
'searchVent': that.searchVent,
- 'filterObj': filterObj
+ 'filterObj': that.filterObj
}, that.preFetchedCollectionLists)
));
} else {
@@ -257,7 +259,7 @@ define([
_.extend({
'value': paramObj,
'initialView': true,
- 'filterObj': filterObj,
+ 'filterObj': that.filterObj,
'searchVent': that.searchVent
}, that.preFetchedCollectionLists)
));
@@ -276,10 +278,7 @@ define([
'views/search/SearchDetailLayoutView'
], function(Header, BusinessCatalogLayoutView, SideNavLayoutView, SearchDetailLayoutView) {
var paramObj = Utils.getUrlState.getQueryParams(),
- filterObj = {
- 'tagFilters': JSON.parse(Utils.localStorage.getValue('tagFilters')),
- 'entityFilters': JSON.parse(Utils.localStorage.getValue('entityFilters'))
- }
+ filterObj = that.filterObj
if (paramObj && paramObj.searchType === "basic") {
if (paramObj.type) {
if (_.has(filterObj.entityFilters, paramObj.type)) {
@@ -310,7 +309,7 @@ define([
_.extend({
'value': paramObj,
'searchVent': that.searchVent,
- 'filterObj': filterObj
+ 'filterObj': that.filterObj
}, that.preFetchedCollectionLists)
));
} else {
@@ -321,7 +320,7 @@ define([
_.extend({
'value': paramObj,
'searchVent': that.searchVent,
- 'filterObj': filterObj,
+ 'filterObj': that.filterObj,
'initialView': (paramObj.type || (paramObj.dslChecked == "true" ? "" : paramObj.tag) || (paramObj.query ? paramObj.query.trim() : "")).length === 0
}, that.preFetchedCollectionLists)
));
@@ -342,4 +341,4 @@ define([
}
});
return AppRouter;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5527afb0/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js b/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
index 013c72e..8f41399 100644
--- a/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
+++ b/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
@@ -96,6 +96,8 @@ define(['require',
that.RTagLayoutView.show(new TagLayoutView({
collection: that.classificationDefCollection,
tag: that.tag,
+ value: that.value,
+ filterObj: that.filterObj,
typeHeaders: that.typeHeaders
}));
});
@@ -128,4 +130,4 @@ define(['require',
},
});
return SideNavLayoutView;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5527afb0/dashboardv2/public/js/views/search/QueryBuilderView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/QueryBuilderView.js b/dashboardv2/public/js/views/search/QueryBuilderView.js
index 468a2d7..ec20734 100644
--- a/dashboardv2/public/js/views/search/QueryBuilderView.js
+++ b/dashboardv2/public/js/views/search/QueryBuilderView.js
@@ -138,13 +138,27 @@ define(['require',
}
}
}
-
_.each(this.attrObj, function(obj) {
var returnObj = that.getObjDef(obj, rules_widgets);
if (returnObj) {
filters.push(returnObj);
}
});
+ filters = _.uniq(filters, 'id');
+ if (rules_widgets) {
+ for (var i = 0; i < rules_widgets.rules.length; i++) {
+ if (!_.find(filters, { id: rules_widgets.rules[i].id })) {
+ var type = (this.tag ? 'tagFilters' : 'entityFilters');
+ var list = JSON.parse(Utils.localStorage.getValue(type));
+ delete list[this.value.tag];
+ list = _.isEmpty(list) ? null : list;
+ Utils.localStorage.setValue(type, JSON.stringify(list));
+ this.filterObj[type] = list;
+ rules_widgets = null;
+ break;
+ }
+ }
+ }
if (filters && !_.isEmpty(filters)) {
this.ui.builder.queryBuilder({
plugins: ['bt-tooltip-errors'],
@@ -178,4 +192,4 @@ define(['require',
}
});
return QueryBuilderView;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5527afb0/dashboardv2/public/js/views/search/SearchLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchLayoutView.js b/dashboardv2/public/js/views/search/SearchLayoutView.js
index cb8844d..76d6f77 100644
--- a/dashboardv2/public/js/views/search/SearchLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchLayoutView.js
@@ -282,32 +282,36 @@ define(['require',
var filtertype = this.attrModal.tag ? 'tagFilters' : 'entityFilters',
rule = this.attrModal.RQueryBuilder.currentView.ui.builder.queryBuilder('getRules'),
result = this.getQueryBuilderParsData(rule);
- if (result && !_.isEmpty(result.criterion)) {
- this.query[this.type][filtertype] = +new Date();
- if (result) {
- var filterObj = this.filterObj ? this.filterObj[filtertype] : null;
- if (!filterObj) {
- filterObj = {};
+
+ if (result) {
+ if (!_.isEmpty(result.criterion)) {
+ this.query[this.type][filtertype] = +new Date();
+ if (result) {
+ var filterObj = this.filterObj ? this.filterObj[filtertype] : null;
+ if (!filterObj) {
+ filterObj = {};
+ }
+ var temp = {}; // IE fix
+ temp[(this.attrModal.tag ? this.value.tag : this.value.type)] = { 'result': result, 'rule': rule };
+ _.extend(filterObj, temp);
+ this.filterObj[filtertype] = filterObj;
+ this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
+ Utils.localStorage.setValue((filtertype), JSON.stringify(filterObj));
+ } else {
+ this.filterObj[filtertype] = null;
+ this.query[this.type][filtertype] = null;
+ this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
+ Utils.localStorage.removeValue(filtertype);
}
- var temp = {}; // IE fix
- temp[(this.attrModal.tag ? this.value.tag : this.value.type)] = { 'result': result, 'rule': rule };
- _.extend(filterObj, temp);
- this.filterObj[filtertype] = filterObj;
- this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
- Utils.localStorage.setValue((filtertype), JSON.stringify(filterObj));
- } else {
- this.filterObj[filtertype] = null;
- this.query[this.type][filtertype] = null;
- this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
- Utils.localStorage.removeValue(filtertype);
+
}
+ this.attrModal.modal.close();
} else {
this.filterObj[filtertype] = null;
this.query[this.type][filtertype] = null;
this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
Utils.localStorage.removeValue(filtertype);
}
- this.attrModal.modal.close();
},
getQueryBuilderParsData: function(obj) {
if (obj) {
@@ -378,17 +382,26 @@ define(['require',
} else if (this.value.dslChecked == "false" && this.dsl == true) {
this.ui.searchType.prop("checked", false).trigger("change");
}
+ this.ui.typeLov.val(this.value.type);
if (this.ui.typeLov.data('select2')) {
- this.ui.typeLov.val(this.value.type).trigger('change');
- } else {
- this.ui.typeLov.val(this.value.type);
+ if (this.ui.typeLov.val() !== this.value.type) {
+ this.value.type = null;
+ this.ui.typeLov.val("").trigger("change");
+ } else {
+ this.ui.typeLov.trigger("change");
+ }
}
if (!this.dsl) {
+ this.ui.tagLov.val(this.value.tag);
if (this.ui.tagLov.data('select2')) {
- this.ui.tagLov.val(this.value.tag).trigger('change');
- } else {
- this.ui.typeLov.val(this.value.tag);
+ // To handle delete scenario.
+ if (this.ui.tagLov.val() !== this.value.tag) {
+ this.value.tag = null;
+ this.ui.tagLov.val("").trigger("change");
+ } else {
+ this.ui.tagLov.trigger("change");
+ }
}
}
this.ui.searchInput.val(this.value.query || "");
@@ -468,7 +481,11 @@ define(['require',
this.ui.typeLov.val("").trigger("change");
this.ui.tagLov.val("").trigger("change");
this.ui.searchInput.val("");
- this.checkForButtonVisiblity()
+ this.checkForButtonVisiblity();
+ Utils.localStorage.removeValue('tagFilters');
+ Utils.localStorage.removeValue('entityFilters');
+ this.filterObj.tagFilters = null;
+ this.filterObj.entityFilters = null;
Utils.setUrl({
url: '#!/search/searchResult',
urlParams: {
@@ -481,4 +498,4 @@ define(['require',
}
});
return SearchLayoutView;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5527afb0/dashboardv2/public/js/views/tag/TagLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/tag/TagLayoutView.js b/dashboardv2/public/js/views/tag/TagLayoutView.js
index b8a1cd1..2714770 100644
--- a/dashboardv2/public/js/views/tag/TagLayoutView.js
+++ b/dashboardv2/public/js/views/tag/TagLayoutView.js
@@ -59,7 +59,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'tag', 'collection', 'typeHeaders'));
+ _.extend(this, _.pick(options, 'tag', 'collection', 'typeHeaders', 'filterObj', 'value'));
},
bindEvents: function() {
var that = this;
@@ -417,10 +417,19 @@ define(['require',
}
that.collection.remove(deleteTagData);
// to update tag list of search tab fetch typeHeaders.
+ var tagList = JSON.parse(Utils.localStorage.getValue('tagFilters'));
+ if (tagList) {
+ delete tagList[that.tagName];
+ }
+ tagList = _.isEmpty(tagList) ? null : tagList;
+ if (that.filterObj['tagFilters'] && that.filterObj['tagFilters'][that.tagName]) {
+ delete that.filterObj['tagFilters'][that.tagName];
+ }
+ Utils.localStorage.setValue('tagFilters', JSON.stringify(tagList));
that.typeHeaders.fetch({ reset: true });
}
});
}
});
return TagLayoutView;
-});
+});
\ No newline at end of file
[35/39] atlas git commit: ATLAS-1981: Cache escaped type-query string
to avoid repeated computation
Posted by ma...@apache.org.
ATLAS-1981: Cache escaped type-query string to avoid repeated computation
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/e0d2cdc2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/e0d2cdc2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/e0d2cdc2
Branch: refs/heads/feature-odf
Commit: e0d2cdc27c32742ebecd24db4cca62dc04dcdf4b
Parents: f511f27
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Jul 21 09:28:44 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 21 09:28:44 2017 -0700
----------------------------------------------------------------------
.../org/apache/atlas/repository/Constants.java | 4 +-
.../atlas/type/AtlasClassificationType.java | 30 ++++++++---
.../org/apache/atlas/type/AtlasEntityType.java | 11 +++-
.../org/apache/atlas/type/AtlasStructType.java | 40 +++++++++++++++
.../apache/atlas/type/AtlasTypeRegistry.java | 4 +-
.../ClassificationSearchProcessor.java | 17 +++---
.../atlas/discovery/EntityDiscoveryService.java | 32 ++++++------
.../atlas/discovery/EntitySearchProcessor.java | 18 ++++---
.../discovery/FullTextSearchProcessor.java | 23 ++++-----
.../apache/atlas/discovery/SearchProcessor.java | 54 ++++----------------
10 files changed, 131 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/common/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
index e8621cf..71d0d8b 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -97,8 +97,8 @@ public final class Constants {
public static final String QUALIFIED_NAME = "Referenceable.qualifiedName";
public static final String TYPE_NAME_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size";
- public static final String INDEX_SEARCH_MAX_TYPES_COUNT = "atlas.graph.index.search.max-types-count";
- public static final String INDEX_SEARCH_MAX_TAGS_COUNT = "atlas.graph.index.search.max-tags-count";
+ public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";
+ public static final String INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.tags.max-query-str-length";
private Constants() {
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
index c1c3add..56c3ed3 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
@@ -45,10 +45,11 @@ public class AtlasClassificationType extends AtlasStructType {
private final AtlasClassificationDef classificationDef;
- private List<AtlasClassificationType> superTypes = Collections.emptyList();
- private Set<String> allSuperTypes = Collections.emptySet();
- private Set<String> allSubTypes = Collections.emptySet();
- private Set<String> typeAndAllSubTypes = Collections.emptySet();
+ private List<AtlasClassificationType> superTypes = Collections.emptyList();
+ private Set<String> allSuperTypes = Collections.emptySet();
+ private Set<String> allSubTypes = Collections.emptySet();
+ private Set<String> typeAndAllSubTypes = Collections.emptySet();
+ private String typeAndAllSubTypesQryStr = "";
public AtlasClassificationType(AtlasClassificationDef classificationDef) {
super(classificationDef);
@@ -108,6 +109,13 @@ public class AtlasClassificationType extends AtlasStructType {
}
}
+ @Override
+ public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ allSubTypes = Collections.unmodifiableSet(allSubTypes);
+ typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
+ typeAndAllSubTypesQryStr = ""; // will be computed on next access
+ }
+
private void addSubType(AtlasClassificationType subType) {
allSubTypes.add(subType.getTypeName());
typeAndAllSubTypes.add(subType.getTypeName());
@@ -119,11 +127,17 @@ public class AtlasClassificationType extends AtlasStructType {
public Set<String> getAllSuperTypes() { return allSuperTypes; }
- public Set<String> getAllSubTypes() {
- return Collections.unmodifiableSet(allSubTypes);
- }
+ public Set<String> getAllSubTypes() { return allSubTypes; }
- public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); }
+ public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
+
+ public String getTypeAndAllSubTypesQryStr() {
+ if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
+ typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
+ }
+
+ return typeAndAllSubTypesQryStr;
+ }
public boolean isSuperTypeOf(AtlasClassificationType classificationType) {
return classificationType != null && allSubTypes.contains(classificationType.getTypeName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index e3005ee..f89c556 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -55,6 +55,7 @@ public class AtlasEntityType extends AtlasStructType {
private Set<String> typeAndAllSuperTypes = Collections.emptySet();
private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
+ private String typeAndAllSubTypesQryStr = "";
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
@@ -150,7 +151,7 @@ public class AtlasEntityType extends AtlasStructType {
allSubTypes = Collections.unmodifiableSet(allSubTypes);
typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
- typeAndAllSuperTypes = Collections.unmodifiableSet(typeAndAllSuperTypes);
+ typeAndAllSubTypesQryStr = ""; // will be computed on next access
relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes);
relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType);
}
@@ -218,6 +219,14 @@ public class AtlasEntityType extends AtlasStructType {
return relationshipAttributesType;
}
+ public String getTypeAndAllSubTypesQryStr() {
+ if (StringUtils.isEmpty(typeAndAllSubTypesQryStr)) {
+ typeAndAllSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndAllSubTypes);
+ }
+
+ return typeAndAllSubTypesQryStr;
+ }
+
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index b390a97..049a537 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -38,6 +38,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* class that implements behaviour of a struct-type.
@@ -713,6 +714,39 @@ public class AtlasStructType extends AtlasType {
return key;
}
+ public static String escapeIndexQueryValue(Set<String> values) {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(BRACE_OPEN_CHAR);
+ for (String value : values) {
+ sb.append(escapeIndexQueryValue(value)).append(SPACE_CHAR);
+ }
+ sb.append(BRACE_CLOSE_CHAR);
+
+ return sb.toString();
+ }
+
+ public static String escapeIndexQueryValue(String value) {
+ String ret = value;
+
+ if (StringUtils.containsAny(value, IDX_QRY_OFFENDING_CHARS)) {
+ boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE_CHAR;
+ boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE_CHAR;
+
+ if (!isQuoteAtStart) {
+ if (!isQuoteAtEnd) {
+ ret = DOUBLE_QUOTE_CHAR + value + DOUBLE_QUOTE_CHAR;
+ } else {
+ ret = DOUBLE_QUOTE_CHAR + value;
+ }
+ } else if (!isQuoteAtEnd) {
+ ret = value + DOUBLE_QUOTE_CHAR;
+ }
+ }
+
+ return ret;
+ }
+
private String getRelationshipEdgeLabel(String relationshipLabel) {
return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel;
}
@@ -730,6 +764,12 @@ public class AtlasStructType extends AtlasType {
new String[] { "%", "_p" },
};
+ private static final char[] IDX_QRY_OFFENDING_CHARS = { '@', '/', ' ' };
+ private static final char BRACE_OPEN_CHAR = '(';
+ private static final char BRACE_CLOSE_CHAR = ')';
+ private static final char DOUBLE_QUOTE_CHAR = '"';
+ private static final char SPACE_CHAR = ' ';
+
public enum AtlasRelationshipEdgeDirection { IN, OUT }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 281422b..e35fb33 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -365,8 +365,8 @@ public class AtlasTypeRegistry {
type.resolveReferencesPhase2(this);
}
- for (AtlasEntityType entityType : registryData.entityDefs.getAllTypes()) {
- entityType.resolveReferencesPhase3(this);
+ for (AtlasType type : registryData.allTypes.getAllTypes()) {
+ type.resolveReferencesPhase3(this);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 745f9d7..dcdcbbb 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -42,23 +42,24 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) {
super(context);
- AtlasClassificationType classificationType = context.getClassificationType();
- FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
- Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
- Set<String> solrAttributes = new HashSet<>();
- Set<String> gremlinAttributes = new HashSet<>();
- Set<String> allAttributes = new HashSet<>();
+ final AtlasClassificationType classificationType = context.getClassificationType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
+ final Set<String> typeAndSubTypes = classificationType.getTypeAndAllSubTypes();
+ final String typeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
+ final Set<String> solrAttributes = new HashSet<>();
+ final Set<String> gremlinAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
processSearchAttributes(classificationType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
// for classification search, if any attribute can't be handled by Solr - switch to all Gremlin
- boolean useSolrSearch = typeAndSubTypes.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
+ boolean useSolrSearch = typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS && CollectionUtils.isEmpty(gremlinAttributes) && canApplySolrFilter(classificationType, filterCriteria, false);
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
- constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index dbe11a6..b183c72 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -87,8 +87,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private final GraphBackedSearchIndexer indexer;
private final SearchTracker searchTracker;
private final int maxResultSetSize;
- private final int maxTypesCountInIdxQuery;
- private final int maxTagsCountInIdxQuery;
+ private final int maxTypesLengthInIdxQuery;
+ private final int maxTagsLengthInIdxQuery;
@Inject
EntityDiscoveryService(MetadataRepository metadataRepository, AtlasTypeRegistry typeRegistry,
@@ -101,8 +101,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
this.typeRegistry = typeRegistry;
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
- this.maxTypesCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
- this.maxTagsCountInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
+ this.maxTypesLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
+ this.maxTagsLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
}
@Override
@@ -490,8 +490,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
- String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesCountInIdxQuery);
- String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsCountInIdxQuery);
+ String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery);
+ String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsLengthInIdxQuery);
StringBuilder queryText = new StringBuilder();
@@ -619,23 +619,23 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
}
- private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesCountInIdxQuery) {
- AtlasClassificationType classification = typeRegistry.getClassificationTypeByName(classificationName);
- Set<String> typeAndSubTypes = classification != null ? classification.getTypeAndAllSubTypes() : null;
+ private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) {
+ AtlasClassificationType type = typeRegistry.getClassificationTypeByName(classificationName);
+ String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
- if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
- return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+ if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
+ return typeAndSubTypesQryStr;
}
return "";
}
- private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesCountInIdxQuery) {
- AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
- Set<String> typeAndSubTypes = type != null ? type.getTypeAndAllSubTypes() : null;
+ private static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesLengthInIdxQuery) {
+ AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
+ String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
- if(CollectionUtils.isNotEmpty(typeAndSubTypes) && typeAndSubTypes.size() <= maxTypesCountInIdxQuery) {
- return String.format("(%s)", StringUtils.join(typeAndSubTypes, " "));
+ if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
+ return typeAndSubTypesQryStr;
}
return "";
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index efcfb7f..385951b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -40,12 +40,13 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
- final AtlasEntityType entityType = context.getEntityType();
- final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
- final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- final Set<String> solrAttributes = new HashSet<>();
- final Set<String> gremlinAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
+ final AtlasEntityType entityType = context.getEntityType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ final String typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
+ final Set<String> solrAttributes = new HashSet<>();
+ final Set<String> gremlinAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
final AtlasClassificationType classificationType = context.getClassificationType();
final boolean filterClassification = classificationType != null && !context.needClassificationProcessor();
@@ -53,13 +54,13 @@ public class EntitySearchProcessor extends SearchProcessor {
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
- final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+ final boolean typeSearchBySolr = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false);
StringBuilder solrQuery = new StringBuilder();
if (typeSearchBySolr) {
- constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ constructTypeTestQuery(solrQuery, typeAndSubTypesQryStr);
}
if (attrSearchBySolr) {
@@ -169,6 +170,7 @@ public class EntitySearchProcessor extends SearchProcessor {
guidQuery.addConditionsFrom(graphQuery);
+ entityVertices.clear();
getVertices(guidQuery.vertices().iterator(), entityVertices);
}
} else {
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 0d1c39b..d556bf1 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.Set;
public class FullTextSearchProcessor extends SearchProcessor {
@@ -50,28 +49,26 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
if (context.getEntityType() != null) {
- Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
+ String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
- if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
- queryString.append(AND_STR);
- appendIndexQueryValue(typeAndSubTypeNames, queryString);
+ if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else {
- LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
- context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+ LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
+ context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
if (context.getClassificationType() != null) {
- Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
+ String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
- if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
- queryString.append(AND_STR);
- appendIndexQueryValue(typeAndSubTypeNames, queryString);
+ if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
+ queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else {
- LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
- context.getClassificationType().getTypeName(), typeAndSubTypeNames.size());
+ LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
+ context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length());
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e0d2cdc2/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 2862165..7e09672 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -39,21 +39,19 @@ import java.util.regex.Pattern;
public abstract class SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class);
- public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
- public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
- public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
- public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
- public static final int MAX_ENTITY_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
- public static final int MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
+ public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
+ public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
+ public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
+ public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
+ public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
+ public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
public static final String AND_STR = " AND ";
public static final String EMPTY_STRING = "";
public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "(";
public static final String BRACE_CLOSE_STR = ")";
- public static final char DOUBLE_QUOTE = '"';
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
- private static final char[] OFFENDING_CHARS = { '@', '/', ' ' }; // This can grow as we discover corner cases
static
{
@@ -181,14 +179,13 @@ public abstract class SearchProcessor {
return ret;
}
- protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
- if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
+ protected void constructTypeTestQuery(StringBuilder solrQuery, String typeAndAllSubTypesQryStr) {
+ if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
if (solrQuery.length() > 0) {
solrQuery.append(AND_STR);
}
- solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":");
- appendIndexQueryValue(typeAndAllSubTypes, solrQuery);
+ solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
}
}
@@ -255,7 +252,7 @@ public abstract class SearchProcessor {
if (OPERATOR_MAP.get(op) != null) {
String qualifiedName = type.getQualifiedAttributeName(attrName);
- ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue(attrVal));
+ ret = String.format(OPERATOR_MAP.get(op), qualifiedName, AtlasStructType.AtlasAttribute.escapeIndexQueryValue(attrVal));
}
} catch (AtlasBaseException ex) {
LOG.warn(ex.getMessage());
@@ -389,16 +386,6 @@ public abstract class SearchProcessor {
return ret;
}
- protected String appendIndexQueryValue(Set<String> values, StringBuilder sb) {
- sb.append(BRACE_OPEN_STR);
- for (String value : values) {
- sb.append(escapeIndexQueryValue(value)).append(SPACE_STRING);
- }
- sb.append(BRACE_CLOSE_STR);
-
- return sb.toString();
- }
-
private static int getApplicationProperty(String propertyName, int defaultValue) {
try {
return ApplicationProperties.get().getInt(propertyName, defaultValue);
@@ -408,25 +395,4 @@ public abstract class SearchProcessor {
return defaultValue;
}
-
- private String escapeIndexQueryValue(String value) {
- String ret = value;
-
- if (StringUtils.containsAny(value, OFFENDING_CHARS)) {
- boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE;
- boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE;
-
- if (!isQuoteAtStart) {
- if (!isQuoteAtEnd) {
- ret = DOUBLE_QUOTE + value + DOUBLE_QUOTE;
- } else {
- ret = DOUBLE_QUOTE + value;
- }
- } else if (!isQuoteAtEnd) {
- ret = value + DOUBLE_QUOTE;
- }
- }
-
- return ret;
- }
}
[03/39] atlas git commit: ATLAS-1907: created RelationshipDefs for
all base models and use it when creating edges for entities
Posted by ma...@apache.org.
ATLAS-1907: created RelationshipDefs for all base models and use it when creating edges for entities
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8fe110c3
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8fe110c3
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8fe110c3
Branch: refs/heads/feature-odf
Commit: 8fe110c3f3c4902d17b082f8431de190fecb0f6f
Parents: b135018
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Tue Jul 11 00:40:02 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 11 05:34:02 2017 -0700
----------------------------------------------------------------------
addons/models/0010-base_model.json | 42 +++-
addons/models/0030-hive_model.json | 102 +++++++-
addons/models/0050-falcon_model.json | 61 ++++-
addons/models/0060-hbase_model.json | 44 +++-
addons/models/0080-storm_model.json | 23 +-
.../java/org/apache/atlas/AtlasErrorCode.java | 7 +-
.../atlas/model/instance/AtlasEntity.java | 26 +-
.../atlas/model/instance/AtlasRelationship.java | 19 +-
.../atlas/model/instance/AtlasStruct.java | 2 +-
.../org/apache/atlas/type/AtlasEntityType.java | 104 ++++++--
.../atlas/type/AtlasRelationshipType.java | 45 +++-
.../org/apache/atlas/type/AtlasStructType.java | 49 ++--
.../java/org/apache/atlas/type/AtlasType.java | 3 +
.../apache/atlas/type/AtlasTypeRegistry.java | 4 +
.../atlas/repository/graph/GraphHelper.java | 55 ++++-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 23 ++
.../store/graph/v1/AtlasGraphUtilsV1.java | 4 +-
.../graph/v1/AtlasRelationshipStoreV1.java | 137 ++++++-----
.../store/graph/v1/EntityGraphMapper.java | 243 ++++++++++++++++---
.../store/graph/v1/EntityGraphRetriever.java | 97 +++++++-
.../graph/v1/AtlasRelationshipStoreV1Test.java | 99 ++++++++
21 files changed, 1031 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/addons/models/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json
index 7f64d85..303f379 100644
--- a/addons/models/0010-base_model.json
+++ b/addons/models/0010-base_model.json
@@ -93,5 +93,45 @@
}
]
}
+ ],
+ "relationshipDefs": [
+ {
+ "name": "dataset_process_inputs",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "endDef1": {
+ "type": "DataSet",
+ "name": "sourceToProcesses",
+ "isContainer": "false",
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "Process",
+ "name": "inputs",
+ "isContainer": "true",
+ "cardinality": "SET",
+ "legacyLabel": "__Process.inputs"
+ },
+ "propagateTags": "NONE"
+ },
+ {
+ "name": "process_dataset_outputs",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "endDef1": {
+ "type": "Process",
+ "name": "outputs",
+ "isContainer": "true",
+ "cardinality": "SET",
+ "legacyLabel": "__Process.outputs"
+ },
+ "endDef2": {
+ "type": "DataSet",
+ "name": "sinkFromProcesses",
+ "isContainer": "false",
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ }
]
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/addons/models/0030-hive_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json
index b359ea8..a795f0f 100644
--- a/addons/models/0030-hive_model.json
+++ b/addons/models/0030-hive_model.json
@@ -521,5 +521,105 @@
}
]
}
+ ],
+ "relationshipDefs": [
+ {
+ "name": "hive_db_tables",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_db",
+ "name": "tables",
+ "isContainer": "true",
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "hive_table",
+ "name": "db",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hive_table.db"
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_table_columns",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_table",
+ "name": "columns",
+ "isContainer": "true",
+ "cardinality": "SET",
+ "legacyLabel": "__hive_table.columns"
+ },
+ "endDef2": {
+ "type": "hive_column",
+ "name": "table",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hive_column.table"
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_table_partitionkeys",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_table",
+ "name": "partitionKeys",
+ "isContainer": "true",
+ "cardinality": "SET",
+ "legacyLabel": "__hive_table.partitionKeys"
+ },
+ "endDef2": {
+ "type": "hive_column",
+ "name": "table",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hive_column.table"
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_table_storagedesc",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_table",
+ "name": "sd",
+ "isContainer": "true",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hive_table.sd"
+ },
+ "endDef2": {
+ "type": "hive_storagedesc",
+ "name": "table",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hive_storagedesc.table"
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_process_column_lineage",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_process",
+ "name": "columnLineages",
+ "isContainer": "true",
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "hive_column_lineage",
+ "name": "query",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hive_column_lineage.query"
+ },
+ "propagateTags": "NONE"
+ }
]
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/addons/models/0050-falcon_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0050-falcon_model.json b/addons/models/0050-falcon_model.json
index b7398ef..7755fa8 100644
--- a/addons/models/0050-falcon_model.json
+++ b/addons/models/0050-falcon_model.json
@@ -143,5 +143,64 @@
}
]
}
+ ],
+ "relationshipDefs": [
+ {
+ "name": "falcon_feed_cluster",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "falcon_feed",
+ "name": "stored-in",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__falcon_feed.stored-in"
+ },
+ "endDef2": {
+ "type": "falcon_cluster",
+ "name": "feeds",
+ "isContainer": "true",
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ },
+ {
+ "name": "falcon_cluster_process",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "falcon_cluster",
+ "name": "processes",
+ "isContainer": "true",
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "falcon_process",
+ "name": "runs-on",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__falcon_process.runs-on"
+ },
+ "propagateTags": "NONE"
+ },
+ {
+ "name": "falcon_cluster_feed_creation",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "falcon_cluster",
+ "name": "feedCreations",
+ "isContainer": "true",
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "falcon_feed_creation",
+ "name": "stored-in",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__falcon_feed_creation.stored-in"
+ },
+ "propagateTags": "NONE"
+ }
]
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/addons/models/0060-hbase_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0060-hbase_model.json b/addons/models/0060-hbase_model.json
index 42fe00c..1d264df 100644
--- a/addons/models/0060-hbase_model.json
+++ b/addons/models/0060-hbase_model.json
@@ -96,5 +96,47 @@
],
"typeVersion": "1.0"
}
+ ],
+ "relationshipDefs": [
+ {
+ "name": "hbase_table_column_families",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hbase_table",
+ "name": "column_families",
+ "isContainer": "true",
+ "cardinality": "SET",
+ "legacyLabel": "__hbase_table.column_families"
+ },
+ "endDef2": {
+ "type": "hbase_column_family",
+ "name": "table",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hbase_column_family.table"
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hbase_column_family_columns",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hbase_column_family",
+ "name": "columns",
+ "isContainer": "true",
+ "cardinality": "SET",
+ "legacyLabel": "__hbase_column_family.columns"
+ },
+ "endDef2": {
+ "type": "hbase_column",
+ "name": "column_family",
+ "isContainer": "false",
+ "cardinality": "SINGLE",
+ "legacyLabel": "__hbase_column.column_family"
+ },
+ "propagateTags": "ONE_TO_TWO"
+ }
]
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/addons/models/0080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json
index 095936e..25360ff 100644
--- a/addons/models/0080-storm_model.json
+++ b/addons/models/0080-storm_model.json
@@ -142,5 +142,26 @@
}
]
}
+ ],
+ "relationshipDefs": [
+ {
+ "name": "storm_topology_nodes",
+ "typeVersion": "1.0",
+ "relationshipCategory": "ASSOCIATION",
+ "endDef1": {
+ "type": "storm_topology",
+ "name": "nodes",
+ "isContainer": "false",
+ "cardinality": "SET",
+ "legacyLabel": "__storm_topology.nodes"
+ },
+ "endDef2": {
+ "type": "storm_node",
+ "name": "topolgies",
+ "isContainer": "false",
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ }
]
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index e8afed1..fd1b004 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -80,13 +80,14 @@ public enum AtlasErrorCode {
RELATIONSHIPDEF_COMPOSITION_MULTIPLE_PARENTS(400, "ATLAS-400-00-033", "COMPOSITION relationshipDef {0} can only have one parent; so cannot have SET cardinality on children"),
RELATIONSHIPDEF_LIST_ON_END(400, "ATLAS-400-00-034", "relationshipDef {0} cannot have a LIST cardinality on an end"),
RELATIONSHIPDEF_INVALID_END_TYPE(400, "ATLAS-400-00-035", "relationshipDef {0} has invalid end type {1}"),
- INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid update for relationshipDef {0}: new end type {1}, existing end type {2}"),
+ INVALID_RELATIONSHIP_END_TYPE(400, "ATLAS-400-00-036", "invalid relationshipDef: {0}: end type 1: {1}, end type 2: {2}"),
RELATIONSHIPDEF_INVALID_END1_UPDATE(400, "ATLAS-400-00-037", "invalid update for relationshipDef {0}: new end1 {1}, existing end1 {2}"),
RELATIONSHIPDEF_INVALID_END2_UPDATE(400, "ATLAS-400-00-038", "invalid update for relationshipDef {0}: new end2 {1}, existing end2 {2}"),
RELATIONSHIPDEF_INVALID_CATEGORY_UPDATE(400, "ATLAS-400-00-039", "invalid update for relationship {0}: new relationshipDef category {1}, existing relationshipDef category {2}"),
RELATIONSHIPDEF_INVALID_NAME_UPDATE(400, "ATLAS-400-00-040", "invalid relationshipDef rename for relationship guid {0}: new name {1}, existing name {2}"),
- RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end1 name. Name must not contain query keywords"),
- RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-020", "{0}: invalid end2 name. Name must not contain query keywords"),
+ RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-041", "{0}: invalid end1 name. Name must not contain query keywords"),
+ RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-042", "{0}: invalid end2 name. Name must not contain query keywords"),
+ RELATIONSHIPDEF_NOT_DEFINED(400, "ATLAS-400-00-043", "No relationshipDef defined between {0} and {1} on attribute: {2}"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS-404-00-002", "Given type guid {0} was invalid"),
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
index 0e277b1..68da6af 100644
--- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
+++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
@@ -70,6 +70,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
private Date updateTime = null;
private Long version = 0L;
+ private Map<String, Object> relationshipAttributes;
private List<AtlasClassification> classifications;
@JsonIgnore
@@ -170,6 +171,25 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this.version = version;
}
+ public Map<String, Object> getRelationshipAttributes() { return relationshipAttributes; }
+
+ public void setRelationshipAttributes(Map<String, Object> relationshipAttributes) {
+ this.relationshipAttributes = relationshipAttributes;
+ }
+
+ public void addRelationshipAttribute(String name, Object value) {
+ Map<String, Object> r = this.relationshipAttributes;
+
+ if (r != null) {
+ r.put(name, value);
+ } else {
+ r = new HashMap<>();
+ r.put(name, value);
+
+ this.relationshipAttributes = r;
+ }
+ }
+
public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
@@ -204,6 +224,9 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
dumpDateField(", createTime=", createTime, sb);
dumpDateField(", updateTime=", updateTime, sb);
sb.append(", version=").append(version);
+ sb.append(", relationshipAttributes=[");
+ dumpObjects(relationshipAttributes, sb);
+ sb.append("]");
sb.append(", classifications=[");
AtlasBaseTypeDef.dumpObjects(classifications, sb);
sb.append(']');
@@ -227,13 +250,14 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
Objects.equals(createTime, that.createTime) &&
Objects.equals(updateTime, that.updateTime) &&
Objects.equals(version, that.version) &&
+ Objects.equals(relationshipAttributes, that.relationshipAttributes) &&
Objects.equals(classifications, that.classifications);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), guid, status, createdBy, updatedBy, createTime, updateTime, version,
- classifications);
+ relationshipAttributes, classifications);
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
index 8d2e7ec..2de9bdf 100644
--- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
+++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
@@ -50,6 +50,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
private String guid = null;
private AtlasObjectId end1 = null;
private AtlasObjectId end2 = null;
+ private String label = null;
private Status status = Status.ACTIVE;
private String createdBy = null;
private String updatedBy = null;
@@ -81,7 +82,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public AtlasRelationship(String typeName, AtlasObjectId end1, AtlasObjectId end2) {
super(typeName);
- init(nextInternalId(), end1, end2, null, null, null, null, null, 0L);
+ init(nextInternalId(), end1, end2, null, null, null, null, null, null, 0L);
}
public AtlasRelationship(String typeName, String attrName, Object attrValue) {
@@ -98,7 +99,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
super(other);
if (other != null) {
- init(other.guid, other.end1, other.end2, other.status, other.createdBy, other.updatedBy,
+ init(other.guid, other.end1, other.end2, other.label, other.status, other.createdBy, other.updatedBy,
other.createTime, other.updateTime, other.version);
}
}
@@ -167,6 +168,10 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public void setEnd2(AtlasObjectId end2) { this.end2 = end2; }
+ public String getLabel() { return label; }
+
+ public void setLabel(String label) { this.label = label; }
+
private static String nextInternalId() {
return "-" + Long.toString(s_nextId.getAndIncrement());
}
@@ -174,15 +179,16 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
public String getRelationshipLabel() { return "r:" + super.getTypeName(); }
private void init() {
- init(nextInternalId(), null, null, null, null, null, null, null, 0L);
+ init(nextInternalId(), null, null, null, null, null, null, null, null, 0L);
}
- private void init(String guid, AtlasObjectId end1, AtlasObjectId end2,
+ private void init(String guid, AtlasObjectId end1, AtlasObjectId end2, String label,
Status status, String createdBy, String updatedBy,
Date createTime, Date updateTime, Long version) {
setGuid(guid);
setEnd1(end1);
setEnd2(end2);
+ setLabel(label);
setStatus(status);
setCreatedBy(createdBy);
setUpdatedBy(updatedBy);
@@ -202,6 +208,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
sb.append("guid='").append(guid).append('\'');
sb.append(", end1=").append(end1);
sb.append(", end2=").append(end2);
+ sb.append(", label='").append(label).append('\'');
sb.append(", status=").append(status);
sb.append(", createdBy='").append(createdBy).append('\'');
sb.append(", updatedBy='").append(updatedBy).append('\'');
@@ -223,6 +230,7 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
return Objects.equals(guid, that.guid) &&
Objects.equals(end1, that.end1) &&
Objects.equals(end2, that.end2) &&
+ Objects.equals(label, that.label) &&
status == that.status &&
Objects.equals(createdBy, that.createdBy) &&
Objects.equals(updatedBy, that.updatedBy) &&
@@ -233,7 +241,8 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
@Override
public int hashCode() {
- return Objects.hash(super.hashCode(), guid, end1, end2, status, createdBy, updatedBy, createTime, updateTime, version);
+ return Objects.hash(super.hashCode(), guid, end1, end2, label, status, createdBy,
+ updatedBy, createTime, updateTime, version);
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
index 7c8ae2d..80f3a66 100644
--- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
+++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasStruct.java
@@ -148,7 +148,7 @@ public class AtlasStruct implements Serializable {
AtlasStruct that = (AtlasStruct) o;
return Objects.equals(typeName, that.typeName) &&
- Objects.equals(attributes, that.attributes);
+ Objects.equals(attributes, that.attributes);
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index 70e3067..e94dd19 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -26,6 +26,7 @@ import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,11 +48,13 @@ public class AtlasEntityType extends AtlasStructType {
private final AtlasEntityDef entityDef;
- private List<AtlasEntityType> superTypes = Collections.emptyList();
- private Set<String> allSuperTypes = Collections.emptySet();
- private Set<String> allSubTypes = Collections.emptySet();
- private Set<String> typeAndAllSubTypes = Collections.emptySet();
- private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
+ private List<AtlasEntityType> superTypes = Collections.emptyList();
+ private Set<String> allSuperTypes = Collections.emptySet();
+ private Set<String> allSubTypes = Collections.emptySet();
+ private Set<String> typeAndAllSubTypes = Collections.emptySet();
+ private Set<String> typeAndAllSuperTypes = Collections.emptySet();
+ private Map<String, AtlasAttribute> relationshipAttributes = Collections.emptyMap();
+ private Map<String, List<AtlasRelationshipType>> relationshipAttributesType = Collections.emptyMap();
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
@@ -89,15 +92,20 @@ public class AtlasEntityType extends AtlasStructType {
}
}
- this.superTypes = Collections.unmodifiableList(s);
- this.allSuperTypes = Collections.unmodifiableSet(allS);
- this.allAttributes = Collections.unmodifiableMap(allA);
- this.uniqAttributes = getUniqueAttributes(this.allAttributes);
- this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
- this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
- this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase2()
+ this.superTypes = Collections.unmodifiableList(s);
+ this.allSuperTypes = Collections.unmodifiableSet(allS);
+ this.allAttributes = Collections.unmodifiableMap(allA);
+ this.uniqAttributes = getUniqueAttributes(this.allAttributes);
+ this.allSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
+ this.typeAndAllSubTypes = new HashSet<>(); // this will be populated in resolveReferencesPhase2()
+ this.relationshipAttributes = new HashMap<>(); // this will be populated in resolveReferencesPhase3()
+ this.relationshipAttributesType = new HashMap<>(); // this will be populated in resolveReferencesPhase3()
this.typeAndAllSubTypes.add(this.getTypeName());
+
+ this.typeAndAllSuperTypes = new HashSet<>(this.allSuperTypes);
+ this.typeAndAllSuperTypes.add(this.getTypeName());
+ this.typeAndAllSuperTypes = Collections.unmodifiableSet(this.typeAndAllSuperTypes);
}
@Override
@@ -110,6 +118,43 @@ public class AtlasEntityType extends AtlasStructType {
}
}
+ @Override
+ public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ for (AtlasAttributeDef attributeDef : getStructDef().getAttributeDefs()) {
+ String attributeName = attributeDef.getName();
+ AtlasType attributeType = typeRegistry.getType(attributeDef.getTypeName());
+ AtlasEntityType attributeEntityType = getReferencedEntityType(attributeType);
+
+ // validate if RelationshipDefs is defined for all entityDefs
+ if (attributeEntityType != null && !hasRelationshipAttribute(attributeName)) {
+ LOG.warn("No RelationshipDef defined between {} and {} on attribute: {}.{}", getTypeName(),
+ attributeEntityType.getTypeName(), getTypeName(), attributeName);
+ }
+ }
+
+ for (String superTypeName : allSuperTypes) {
+ AtlasEntityType superType = typeRegistry.getEntityTypeByName(superTypeName);
+
+ Map<String, AtlasAttribute> superTypeRelationshipAttributes = superType.getRelationshipAttributes();
+
+ if (MapUtils.isNotEmpty(superTypeRelationshipAttributes)) {
+ relationshipAttributes.putAll(superTypeRelationshipAttributes);
+ }
+
+ Map<String, List<AtlasRelationshipType>> superTypeRelationshipAttributesType = superType.getRelationshipAttributesType();
+
+ if (MapUtils.isNotEmpty(superTypeRelationshipAttributesType)) {
+ relationshipAttributesType.putAll(superTypeRelationshipAttributesType);
+ }
+ }
+
+ allSubTypes = Collections.unmodifiableSet(allSubTypes);
+ typeAndAllSubTypes = Collections.unmodifiableSet(typeAndAllSubTypes);
+ typeAndAllSuperTypes = Collections.unmodifiableSet(typeAndAllSuperTypes);
+ relationshipAttributes = Collections.unmodifiableMap(relationshipAttributes);
+ relationshipAttributesType = Collections.unmodifiableMap(relationshipAttributesType);
+ }
+
public Set<String> getSuperTypes() {
return entityDef.getSuperTypes();
}
@@ -118,9 +163,11 @@ public class AtlasEntityType extends AtlasStructType {
return allSuperTypes;
}
- public Set<String> getAllSubTypes() { return Collections.unmodifiableSet(allSubTypes); }
+ public Set<String> getAllSubTypes() { return allSubTypes; }
- public Set<String> getTypeAndAllSubTypes() { return Collections.unmodifiableSet(typeAndAllSubTypes); }
+ public Set<String> getTypeAndAllSubTypes() { return typeAndAllSubTypes; }
+
+ public Set<String> getTypeAndAllSuperTypes() { return typeAndAllSuperTypes; }
public boolean isSuperTypeOf(AtlasEntityType entityType) {
return entityType != null && allSubTypes.contains(entityType.getTypeName());
@@ -142,12 +189,37 @@ public class AtlasEntityType extends AtlasStructType {
return StringUtils.isNotEmpty(entityTypeName) && allSuperTypes.contains(entityTypeName);
}
- public Map<String, AtlasAttribute> getRelationshipAttributes() { return Collections.unmodifiableMap(relationshipAttributes); }
+ public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
- public void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
+ // this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
+ void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
relationshipAttributes.put(attributeName, attribute);
}
+ // this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
+ void addRelationshipAttributeType(String attributeName, AtlasRelationshipType relationshipType) {
+ List<AtlasRelationshipType> relationshipTypes = relationshipAttributesType.get(attributeName);
+
+ if (relationshipTypes == null) {
+ relationshipTypes = new ArrayList<>();
+ relationshipAttributesType.put(attributeName, relationshipTypes);
+ }
+
+ relationshipTypes.add(relationshipType);
+ }
+
+ public List<AtlasRelationshipType> getRelationshipAttributeType(String attributeName) {
+ return relationshipAttributesType.get(attributeName);
+ }
+
+ public Map<String, List<AtlasRelationshipType>> getRelationshipAttributesType() {
+ return relationshipAttributesType;
+ }
+
+ public boolean hasRelationshipAttribute(String attributeName) {
+ return relationshipAttributes.containsKey(attributeName);
+ }
+
@Override
public AtlasEntity createDefaultValue() {
AtlasEntity ret = new AtlasEntity(entityDef.getName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
index 49a9493..841b66f 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
@@ -90,9 +90,26 @@ public class AtlasRelationshipType extends AtlasStructType {
public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferencesPhase2(typeRegistry);
- addRelationshipAttributeToEndType(relationshipDef.getEndDef1(), end1Type, end2Type.getTypeName(), typeRegistry);
+ AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
+ String relationshipLabel = null;
- addRelationshipAttributeToEndType(relationshipDef.getEndDef2(), end2Type, end1Type.getTypeName(), typeRegistry);
+ // if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
+ // if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
+ // if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
+ if (!endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
+ relationshipLabel = relationshipDef.getName();
+
+ } else if (endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
+ relationshipLabel = endDef1.getLegacyLabel();
+
+ } else if (!endDef1.hasLegacyRelation() && endDef2.hasLegacyRelation()) {
+ relationshipLabel = endDef2.getLegacyLabel();
+ }
+
+ addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel);
+
+ addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel);
}
@Override
@@ -198,10 +215,8 @@ public class AtlasRelationshipType extends AtlasStructType {
}
}
- private void addRelationshipAttributeToEndType(AtlasRelationshipEndDef endDef,
- AtlasEntityType entityType,
- String attrTypeName,
- AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ private void addRelationshipAttributeToEndType(AtlasRelationshipEndDef endDef, AtlasEntityType entityType, String attrTypeName,
+ AtlasTypeRegistry typeRegistry, String relationshipLabel) throws AtlasBaseException {
String attrName = (endDef != null) ? endDef.getName() : null;
@@ -211,15 +226,29 @@ public class AtlasRelationshipType extends AtlasStructType {
AtlasAttribute attribute = entityType.getAttribute(attrName);
+ // if relationshipLabel is null, then legacyLabel is mentioned at both ends,
+ // use the respective end's legacyLabel as relationshipLabel
+ if (relationshipLabel == null) {
+ relationshipLabel = endDef.getLegacyLabel();
+ }
+
if (attribute == null) { //attr doesn't exist in type - is a new relationship attribute
if (endDef.getCardinality() == Cardinality.SET) {
attrTypeName = AtlasBaseTypeDef.getArrayTypeName(attrTypeName);
}
- attribute = new AtlasAttribute(entityType, new AtlasAttributeDef(attrName, attrTypeName), typeRegistry.getType(attrTypeName));
+ attribute = new AtlasAttribute(entityType, new AtlasAttributeDef(attrName, attrTypeName),
+ typeRegistry.getType(attrTypeName), relationshipLabel);
+
+ } else {
+ // attribute already exists (legacy attribute which is also a relationship attribute)
+ // add relationshipLabel information to existing attribute
+ attribute.setRelationshipEdgeLabel(relationshipLabel);
}
entityType.addRelationshipAttribute(attrName, attribute);
+
+ entityType.addRelationshipAttributeType(attrName, this);
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index c2e0be5..f97d767 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -28,7 +28,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
-import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
@@ -170,9 +169,10 @@ public class AtlasStructType extends AtlasType {
continue;
}
// Set the inverse reference attribute.
- AtlasType referencedType = typeRegistry.getType(attribute.getAttributeDef().getTypeName());
+ AtlasType referencedType = typeRegistry.getType(attribute.getAttributeDef().getTypeName());
AtlasEntityType referencedEntityType = getReferencedEntityType(referencedType);
- AtlasAttribute inverseReference = referencedEntityType.getAttribute(attribute.getInverseRefAttributeName());
+ AtlasAttribute inverseReference = referencedEntityType.getAttribute(attribute.getInverseRefAttributeName());
+
attribute.setInverseRefAttribute(inverseReference);
}
}
@@ -574,7 +574,7 @@ public class AtlasStructType extends AtlasType {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, structDef.getName());
}
- private AtlasEntityType getReferencedEntityType(AtlasType type) {
+ AtlasEntityType getReferencedEntityType(AtlasType type) {
if (type instanceof AtlasArrayType) {
type = ((AtlasArrayType)type).getElementType();
}
@@ -609,16 +609,17 @@ public class AtlasStructType extends AtlasType {
private final boolean isOwnedRef;
private final String inverseRefAttributeName;
private AtlasAttribute inverseRefAttribute;
-
- public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
- this.definedInType = definedInType;
- this.attributeDef = attrDef;
- this.attributeType = attributeType.getTypeForAttribute();
- this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
- this.vertexPropertyName = encodePropertyKey(this.qualifiedName);
-
- boolean isOwnedRef = false;
- String inverseRefAttribute = null;
+ private String relationshipEdgeLabel;
+
+ public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipLabel) {
+ this.definedInType = definedInType;
+ this.attributeDef = attrDef;
+ this.attributeType = attributeType.getTypeForAttribute();
+ this.qualifiedName = getQualifiedAttributeName(definedInType.getStructDef(), attributeDef.getName());
+ this.vertexPropertyName = encodePropertyKey(this.qualifiedName);
+ this.relationshipEdgeLabel = getRelationshipEdgeLabel(relationshipLabel);
+ boolean isOwnedRef = false;
+ String inverseRefAttribute = null;
if (CollectionUtils.isNotEmpty(attributeDef.getConstraints())) {
for (AtlasConstraintDef constraint : attributeDef.getConstraints()) {
@@ -636,10 +637,14 @@ public class AtlasStructType extends AtlasType {
}
}
- this.isOwnedRef = isOwnedRef;
+ this.isOwnedRef = isOwnedRef;
this.inverseRefAttributeName = inverseRefAttribute;
}
+ public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
+ this(definedInType, attrDef, attributeType, null);
+ }
+
public AtlasStructType getDefinedInType() { return definedInType; }
public AtlasStructDef getDefinedInDef() { return definedInType.getStructDef(); }
@@ -666,7 +671,15 @@ public class AtlasStructType extends AtlasType {
public AtlasAttribute getInverseRefAttribute() { return inverseRefAttribute; }
- public void setInverseRefAttribute(AtlasAttribute inverseAttr) { inverseRefAttribute = inverseAttr; };
+ public void setInverseRefAttribute(AtlasAttribute inverseAttr) { inverseRefAttribute = inverseAttr; }
+
+ public String getRelationshipEdgeLabel() { return relationshipEdgeLabel; }
+
+ public void setRelationshipEdgeLabel(String relationshipEdgeLabel) { this.relationshipEdgeLabel = relationshipEdgeLabel; }
+
+ public static String getEdgeLabel(String property) {
+ return "__" + property;
+ }
public static String encodePropertyKey(String key) {
if (StringUtils.isBlank(key)) {
@@ -692,6 +705,10 @@ public class AtlasStructType extends AtlasType {
return key;
}
+ private String getRelationshipEdgeLabel(String relationshipLabel) {
+ return (relationshipLabel == null) ? getEdgeLabel(vertexPropertyName) : relationshipLabel;
+ }
+
private static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
final String typeName = structDef.getName();
return attrName.contains(".") ? attrName : String.format("%s.%s", typeName, attrName);
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/type/AtlasType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
index 86072fe..c99eb7f 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
@@ -57,6 +57,9 @@ public abstract class AtlasType {
public void resolveReferencesPhase2(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
}
+ public void resolveReferencesPhase3(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ }
+
public String getTypeName() { return typeName; }
public TypeCategory getTypeCategory() { return typeCategory; }
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 29fae1c..281422b 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -364,6 +364,10 @@ public class AtlasTypeRegistry {
for (AtlasType type : registryData.allTypes.getAllTypes()) {
type.resolveReferencesPhase2(this);
}
+
+ for (AtlasEntityType entityType : registryData.entityDefs.getAllTypes()) {
+ entityType.resolveReferencesPhase3(this);
+ }
}
public void clear() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 329dd7a..c5e32d8 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -27,6 +27,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.model.instance.AtlasRelationship;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -36,6 +37,8 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
@@ -345,6 +348,10 @@ public final class GraphHelper {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
}
+ public Iterator<AtlasEdge> getBothEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
+ return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.BOTH, edgeLabel);
+ }
+
/**
* Returns the active edge for the given edge label.
* If the vertex is deleted and there is no active edge, it returns the latest deleted edge
@@ -1223,4 +1230,50 @@ public final class GraphHelper {
return condition.toString();
}
-}
+
+ /**
+ * Get relationshipDef name from entityType using relationship attribute.
+ * if more than one relationDefs are returned for an attribute.
+ * e.g. hive_column.table
+ *
+ * hive_table.columns -> hive_column.table
+ * hive_table.partitionKeys -> hive_column.table
+ *
+ * resolve by comparing all incoming edges typename with relationDefs name returned for an attribute
+ * to pick the right relationshipDef name
+ */
+ public String getRelationshipDefName(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
+ AtlasRelationshipDef relationshipDef = getRelationshipDef(entityVertex, entityType, attributeName);
+
+ return (relationshipDef != null) ? relationshipDef.getName() : null;
+ }
+
+ public AtlasRelationshipDef getRelationshipDef(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
+ List<AtlasRelationshipType> relationshipTypes = entityType.getRelationshipAttributeType(attributeName);
+ AtlasRelationshipDef ret = null;
+
+ if (relationshipTypes.size() > 1) {
+ Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
+
+ while (iter.hasNext() && ret == null) {
+ String edgeTypeName = AtlasGraphUtilsV1.getTypeName(iter.next());
+
+ for (AtlasRelationshipType relationType : relationshipTypes) {
+ AtlasRelationshipDef relationshipDef = relationType.getRelationshipDef();
+
+ if (StringUtils.equals(edgeTypeName, relationshipDef.getName())) {
+ ret = relationshipDef;
+
+ break;
+ }
+ }
+ }
+
+ } else {
+ //relationshipTypes will have at least one relationshipDef
+ ret = relationshipTypes.get(0).getRelationshipDef();
+ }
+
+ return ret;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 4530f51..b67c50d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -24,6 +24,7 @@ import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
@@ -164,6 +165,14 @@ public class AtlasTypeDefStoreInitializer {
}
}
+ if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
+ for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
+ if (!typeRegistry.isRegisteredType(relationshipDef.getName())) {
+ typesToCreate.getRelationshipDefs().add(relationshipDef);
+ }
+ }
+ }
+
return typesToCreate;
}
@@ -234,6 +243,20 @@ public class AtlasTypeDefStoreInitializer {
}
}
+ if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) {
+ for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) {
+ AtlasRelationshipDef oldRelationshipDef = typeRegistry.getRelationshipDefByName(relationshipDef.getName());
+
+ if (oldRelationshipDef == null) {
+ continue;
+ }
+
+ if (updateTypeAttributes(oldRelationshipDef, relationshipDef)) {
+ typesToUpdate.getRelationshipDefs().add(relationshipDef);
+ }
+ }
+ }
+
return typesToUpdate;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
index 7b3f1e6..cd9a47a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
@@ -76,8 +76,8 @@ public class AtlasGraphUtilsV1 {
return vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
}
- public static String getTypeName(AtlasVertex instanceVertex) {
- return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
+ public static String getTypeName(AtlasElement element) {
+ return element.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
}
public static String getEdgeLabel(String fromNode, String toNode) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
index 8621233..8d9e4be 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
@@ -23,7 +23,6 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
-import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
@@ -47,8 +46,10 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
@Component
@@ -75,18 +76,16 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
validateRelationship(relationship);
- String relationshipLabel = relationship.getRelationshipLabel();
- AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
- AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
-
+ AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
+ AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
AtlasRelationship ret;
// create relationship between two vertex
try {
- AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationshipLabel);
+ AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship);
if (relationshipEdge == null) {
- relationshipEdge = createRelationEdge(end1Vertex, end2Vertex, relationship);
+ relationshipEdge = createRelationshipEdge(end1Vertex, end2Vertex, relationship);
AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
@@ -99,9 +98,6 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
}
}
- // create legacy edges if mentioned in relationDef
- createLegacyEdges(relationType.getRelationshipDef(), end1Vertex, end2Vertex);
-
ret = mapEdgeToAtlasRelationship(relationshipEdge);
} else {
@@ -181,49 +177,34 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
}
String relationshipName = relationship.getTypeName();
+ String end1TypeName = getTypeNameFromObjectId(relationship.getEnd1());
+ String end2TypeName = getTypeNameFromObjectId(relationship.getEnd2());
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationshipName);
if (relationshipType == null) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "unknown relationship type'" + relationshipName + "'");
}
- AtlasObjectId end1 = relationship.getEnd1();
-
- if (end1 == null) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "end1 is null");
+ if (relationship.getEnd1() == null || relationship.getEnd2() == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "end1/end2 is null");
}
- String end1TypeName = end1.getTypeName();
-
- if (StringUtils.isBlank(end1TypeName)) {
- end1TypeName = AtlasGraphUtilsV1.getTypeNameFromGuid(end1.getGuid());
- }
+ if (!relationshipType.getEnd1Type().isTypeOrSuperTypeOf(end1TypeName) &&
+ !relationshipType.getEnd2Type().isTypeOrSuperTypeOf(end1TypeName)) {
- if (!relationshipType.getEnd1Type().isTypeOrSuperTypeOf(end1TypeName)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_END_TYPE, relationshipName,
- relationshipType.getEnd1Type().getTypeName(), end1TypeName);
+ relationshipType.getEnd2Type().getTypeName(), end1TypeName);
}
- AtlasObjectId end2 = relationship.getEnd2();
-
- if (end2 == null) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "end2 is null");
- }
-
- String end2TypeName = end2.getTypeName();
-
- if (StringUtils.isBlank(end2TypeName)) {
- end2TypeName = AtlasGraphUtilsV1.getTypeNameFromGuid(end2.getGuid());
- }
+ if (!relationshipType.getEnd2Type().isTypeOrSuperTypeOf(end2TypeName) &&
+ !relationshipType.getEnd1Type().isTypeOrSuperTypeOf(end2TypeName)) {
- if (!relationshipType.getEnd2Type().isTypeOrSuperTypeOf(end2TypeName)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_END_TYPE, relationshipName,
- relationshipType.getEnd2Type().getTypeName(), end2TypeName);
+ relationshipType.getEnd1Type().getTypeName(), end2TypeName);
}
- validateEnd(end1);
-
- validateEnd(end2);
+ validateEnd(relationship.getEnd1());
+ validateEnd(relationship.getEnd2());
validateAndNormalize(relationship);
}
@@ -267,15 +248,16 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
type.getNormalizedValue(relationship);
}
- private AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String relationshipLabel) {
- AtlasEdge ret = graphHelper.getEdgeForLabel(fromVertex, relationshipLabel);
+ private AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
+ String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship);
+ AtlasEdge ret = graphHelper.getEdgeForLabel(fromVertex, relationshipLabel);
if (ret != null) {
AtlasVertex inVertex = ret.getInVertex();
if (inVertex != null) {
if (!StringUtils.equals(AtlasGraphUtilsV1.getIdFromVertex(inVertex),
- AtlasGraphUtilsV1.getIdFromVertex(toVertex))) {
+ AtlasGraphUtilsV1.getIdFromVertex(toVertex))) {
ret = null;
}
}
@@ -284,7 +266,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
return ret;
}
- private int getRelationVersion(AtlasRelationship relationship) {
+ private int getRelationshipVersion(AtlasRelationship relationship) {
Long ret = relationship != null ? relationship.getVersion() : null;
return (ret != null) ? ret.intValue() : DEFAULT_RELATIONSHIP_VERSION;
@@ -305,36 +287,61 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
return ret;
}
- private void createLegacyEdges(AtlasRelationshipDef relationshipDef, AtlasVertex fromVertex, AtlasVertex toVertex) throws RepositoryException {
- if (relationshipDef != null) {
- AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
- AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
+ private AtlasEdge createRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship)
+ throws RepositoryException {
- if (endDef1 != null && endDef1.hasLegacyRelation()) {
- graphHelper.getOrCreateEdge(fromVertex, toVertex, endDef1.getLegacyLabel());
- }
+ String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship);
+ AtlasEdge ret = graphHelper.getOrCreateEdge(fromVertex, toVertex, relationshipLabel);
- if (endDef2 != null && endDef2.hasLegacyRelation()) {
- graphHelper.getOrCreateEdge(toVertex, fromVertex, endDef2.getLegacyLabel());
- }
- }
- }
-
- private AtlasEdge createRelationEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) throws RepositoryException {
- AtlasEdge ret = graphHelper.getOrCreateEdge(fromVertex, toVertex, relationship.getRelationshipLabel());
-
- // add additional properties to edge
+ // map additional properties to relationship edge
if (ret != null) {
final String guid = UUID.randomUUID().toString();
AtlasGraphUtilsV1.setProperty(ret, Constants.ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
AtlasGraphUtilsV1.setProperty(ret, Constants.GUID_PROPERTY_KEY, guid);
- AtlasGraphUtilsV1.setProperty(ret, Constants.VERSION_PROPERTY_KEY, getRelationVersion(relationship));
+ AtlasGraphUtilsV1.setProperty(ret, Constants.VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
+ }
+
+ return ret;
+ }
+
+ private String getRelationshipEdgeLabel(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
+
+ String ret = relationship.getRelationshipLabel();
+ AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
+ AtlasRelationshipEndDef endDef1 = relationshipType.getRelationshipDef().getEndDef1();
+ AtlasRelationshipEndDef endDef2 = relationshipType.getRelationshipDef().getEndDef2();
+ Set<String> fromVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV1.getTypeName(fromVertex));
+ Set<String> toVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV1.getTypeName(toVertex));
+
+ // validate entity type and all its supertypes contains relationshipDefs end type
+ // e.g. [ hive_process -> hive_table] -> [ Process -> DataSet ]
+ if (fromVertexTypes.contains(endDef1.getType()) && toVertexTypes.contains(endDef2.getType())) {
+ String attributeName = endDef1.getName();
+ AtlasAttribute endAttribute = relationshipType.getEnd1Type().getAttribute(attributeName);
+
+ if (endAttribute != null) {
+ ret = endAttribute.getRelationshipEdgeLabel();
+ }
+
+ } else if (fromVertexTypes.contains(endDef2.getType()) && toVertexTypes.contains(endDef1.getType())) {
+ String attributeName = endDef2.getName();
+ AtlasAttribute endAttribute = relationshipType.getEnd2Type().getAttribute(attributeName);
+
+ if (endAttribute != null) {
+ ret = endAttribute.getRelationshipEdgeLabel();
+ }
}
return ret;
}
+ public Set<String> getTypeAndAllSuperTypes(String entityTypeName) {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName);
+
+ return (entityType != null) ? entityType.getTypeAndAllSuperTypes() : new HashSet<String>();
+ }
+
private AtlasRelationship mapEdgeToAtlasRelationship(AtlasEdge edge) throws AtlasBaseException {
AtlasRelationship ret = new AtlasRelationship();
@@ -368,6 +375,8 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
relationship.setEnd1(new AtlasObjectId(GraphHelper.getGuid(end1Vertex), GraphHelper.getTypeName(end1Vertex)));
relationship.setEnd2(new AtlasObjectId(GraphHelper.getGuid(end2Vertex), GraphHelper.getTypeName(end2Vertex)));
+ relationship.setLabel(edge.getLabel());
+
return relationship;
}
@@ -388,4 +397,14 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
relationship.setAttribute(attribute.getName(), attrValue);
}
}
+
+ private String getTypeNameFromObjectId(AtlasObjectId objectId) {
+ String typeName = objectId.getTypeName();
+
+ if (StringUtils.isBlank(typeName)) {
+ typeName = AtlasGraphUtilsV1.getTypeNameFromGuid(objectId.getGuid());
+ }
+
+ return typeName;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index ebf6a20..d5c1e86 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -27,6 +27,7 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
@@ -37,6 +38,7 @@ import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
@@ -67,17 +69,19 @@ import static org.apache.atlas.repository.graph.GraphHelper.string;
public class EntityGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class);
- private final GraphHelper graphHelper = GraphHelper.getInstance();
- private final AtlasGraph graph;
- private final DeleteHandlerV1 deleteHandler;
- private final AtlasTypeRegistry typeRegistry;
-
+ private final GraphHelper graphHelper = GraphHelper.getInstance();
+ private final AtlasGraph graph;
+ private final DeleteHandlerV1 deleteHandler;
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasRelationshipStore relationshipStore;
@Inject
- public EntityGraphMapper(DeleteHandlerV1 deleteHandler, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) {
- this.deleteHandler = deleteHandler;
- this.typeRegistry = typeRegistry;
- this.graph = atlasGraph;
+ public EntityGraphMapper(DeleteHandlerV1 deleteHandler, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
+ AtlasRelationshipStore relationshipStore) {
+ this.deleteHandler = deleteHandler;
+ this.typeRegistry = typeRegistry;
+ this.graph = atlasGraph;
+ this.relationshipStore = relationshipStore;
}
public AtlasVertex createVertex(AtlasEntity entity) {
@@ -305,9 +309,15 @@ public class EntityGraphMapper {
}
case OBJECT_ID_TYPE: {
- String edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexProperty());
- AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel);
- AtlasEdge newEdge = null;
+ String edgeLabel = ctx.getAttribute().getRelationshipEdgeLabel();
+
+ // legacy case - if relationship attribute doesn't exist, use legacy edge label.
+ if (StringUtils.isEmpty(edgeLabel)) {
+ edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexProperty());
+ }
+
+ AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel);
+ AtlasEdge newEdge = null;
if (ctx.getValue() != null) {
AtlasEntityType instanceType = getInstanceType(ctx.getValue());
@@ -316,12 +326,14 @@ public class EntityGraphMapper {
ctx.setElementType(instanceType);
ctx.setExistingEdge(edge);
- newEdge = mapObjectIdValue(ctx, context);
+ newEdge = mapObjectIdValueUsingRelationship(ctx, context);
+
if (ctx.getAttribute().getInverseRefAttribute() != null) {
- // Update the inverse reference on the target entity
- addInverseReference(ctx, ctx.getAttribute().getInverseRefAttribute(), newEdge);
+ // Update the inverse reference using relationship on the target entity
+ addInverseReference(ctx.getAttribute().getInverseRefAttribute(), newEdge);
}
}
+
if (currentEdge != null && !currentEdge.equals(newEdge)) {
deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), ctx.getAttribute().isOwnedRef(), true);
}
@@ -340,21 +352,15 @@ public class EntityGraphMapper {
}
}
- private void addInverseReference(AttributeMutationContext ctx, AtlasAttribute inverseAttribute, AtlasEdge edge) throws AtlasBaseException {
+ private void addInverseReference(AtlasAttribute inverseAttribute, AtlasEdge edge) throws AtlasBaseException {
+ AtlasStructType inverseType = inverseAttribute.getDefinedInType();
+ AtlasVertex inverseVertex = edge.getInVertex();
+ String inverseEdgeLabel = inverseAttribute.getRelationshipEdgeLabel();
+ AtlasEdge inverseEdge = graphHelper.getEdgeForLabel(inverseVertex, inverseEdgeLabel);
+ String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(inverseType, inverseAttribute.getName());
- AtlasStructType inverseType = inverseAttribute.getDefinedInType();
- String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(inverseType, inverseAttribute.getName());
- AtlasVertex vertex = edge.getOutVertex();
- AtlasVertex inverseVertex = edge.getInVertex();
- String inverseEdgeLabel = AtlasGraphUtilsV1.getEdgeLabel(propertyName);
- AtlasEdge inverseEdge = graphHelper.getEdgeForLabel(inverseVertex, inverseEdgeLabel);
-
- AtlasEdge newEdge;
- try {
- newEdge = graphHelper.getOrCreateEdge(inverseVertex, vertex, inverseEdgeLabel);
- } catch (RepositoryException e) {
- throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
- }
+ // create new inverse reference
+ AtlasEdge newEdge = createInverseReferenceUsingRelationship(inverseAttribute, edge);
boolean inverseUpdated = true;
switch (inverseAttribute.getAttributeType().getTypeCategory()) {
@@ -363,7 +369,7 @@ public class EntityGraphMapper {
if (!inverseEdge.equals(newEdge)) {
// Disconnect old reference
deleteHandler.deleteEdgeReference(inverseEdge, inverseAttribute.getAttributeType().getTypeCategory(),
- inverseAttribute.isOwnedRef(), true);
+ inverseAttribute.isOwnedRef(), true);
}
else {
// Edge already exists for this attribute between these vertices.
@@ -401,6 +407,60 @@ public class EntityGraphMapper {
}
}
+ private AtlasEdge createInverseReferenceUsingRelationship(AtlasAttribute inverseAttribute, AtlasEdge edge) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> createInverseReferenceUsingRelationship()");
+ }
+
+ String inverseAttributeName = inverseAttribute.getName();
+ AtlasType inverseAttributeType = inverseAttribute.getDefinedInType();
+ AtlasVertex inverseVertex = edge.getInVertex();
+ AtlasVertex vertex = edge.getOutVertex();
+ AtlasEdge ret;
+
+ if (inverseAttributeType instanceof AtlasEntityType) {
+ AtlasEntityType entityType = (AtlasEntityType) inverseAttributeType;
+
+ if (entityType.hasRelationshipAttribute(inverseAttributeName)) {
+ ret = createRelationship(inverseVertex, vertex, inverseAttribute.getRelationshipEdgeLabel());
+
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No RelationshipDef defined between {} and {} on attribute: {}", inverseAttributeType,
+ AtlasGraphUtilsV1.getTypeName(vertex), inverseAttributeName);
+ }
+ // if no RelationshipDef found, use legacy way to create edges
+ ret = createInverseReference(inverseAttribute, (AtlasStructType) inverseAttributeType, inverseVertex, vertex);
+ }
+ } else {
+ // inverseAttribute not of type AtlasEntityType, use legacy way to create edges
+ ret = createInverseReference(inverseAttribute, (AtlasStructType) inverseAttributeType, inverseVertex, vertex);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== createInverseReferenceUsingRelationship()");
+ }
+
+ return ret;
+ }
+
+ // legacy method to create edges for inverse reference
+ private AtlasEdge createInverseReference(AtlasAttribute inverseAttribute, AtlasStructType inverseAttributeType,
+ AtlasVertex inverseVertex, AtlasVertex vertex) throws AtlasBaseException {
+
+ String propertyName = AtlasGraphUtilsV1.getQualifiedAttributePropertyKey(inverseAttributeType, inverseAttribute.getName());
+ String inverseEdgeLabel = AtlasGraphUtilsV1.getEdgeLabel(propertyName);
+ AtlasEdge ret;
+
+ try {
+ ret = graphHelper.getOrCreateEdge(inverseVertex, vertex, inverseEdgeLabel);
+
+ } catch (RepositoryException e) {
+ throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
+ }
+
+ return ret;
+ }
private Object mapPrimitiveValue(AttributeMutationContext ctx) {
AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), ctx.getValue());
@@ -492,6 +552,63 @@ public class EntityGraphMapper {
return ret;
}
+ private AtlasEdge mapObjectIdValueUsingRelationship(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> mapObjectIdValueUsingRelationship({})", ctx);
+ }
+
+ AtlasVertex attributeVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue()));
+ AtlasVertex entityVertex = ctx.getReferringVertex();
+ AtlasEdge ret;
+
+ if (attributeVertex == null) {
+ AtlasObjectId objectId = getObjectId(ctx.getValue());
+
+ attributeVertex = (objectId != null) ? context.getDiscoveryContext().getResolvedEntityVertex(objectId) : null;
+ }
+
+ if (attributeVertex == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, (ctx.getValue() == null ? null : ctx.getValue().toString()));
+ }
+
+ String attributeName = ctx.getAttribute().getName();
+ AtlasType type = typeRegistry.getType(AtlasGraphUtilsV1.getTypeName(entityVertex));
+
+ if (type instanceof AtlasEntityType) {
+ AtlasEntityType entityType = (AtlasEntityType) type;
+
+ // use relationship to create/update edges
+ if (entityType.hasRelationshipAttribute(attributeName)) {
+ if (ctx.getCurrentEdge() != null) {
+ ret = updateRelationship(ctx.getCurrentEdge(), attributeVertex);
+
+ } else {
+ String relationshipName = graphHelper.getRelationshipDefName(entityVertex, entityType, attributeName);
+ ret = createRelationship(entityVertex, attributeVertex, relationshipName);
+ }
+
+ } else {
+ // use legacy way to create/update edges
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No RelationshipDef defined between {} and {} on attribute: {}", AtlasGraphUtilsV1.getTypeName(entityVertex),
+ AtlasGraphUtilsV1.getTypeName(attributeVertex), attributeName);
+ }
+
+ ret = mapObjectIdValue(ctx, context);
+ }
+
+ } else {
+ // if type is StructType having objectid as attribute
+ ret = mapObjectIdValue(ctx, context);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== mapObjectIdValueUsingRelationship({})", ctx);
+ }
+
+ return ret;
+ }
+
private Map<String, Object> mapMapValue(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> mapMapValue({})", ctx);
@@ -536,7 +653,7 @@ public class EntityGraphMapper {
// update the inverse reference value.
if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) {
AtlasEdge newEdge = (AtlasEdge) newEntry;
- addInverseReference(mapCtx, inverseRefAttribute, newEdge);
+ addInverseReference(inverseRefAttribute, newEdge);
}
}
}
@@ -568,14 +685,14 @@ public class EntityGraphMapper {
LOG.debug("==> mapArrayValue({})", ctx);
}
- AtlasAttribute attribute = ctx.getAttribute();
- List newElements = (List) ctx.getValue();
- AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType();
- AtlasType elementType = arrType.getElementType();
- List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty());
- boolean isReference = AtlasGraphUtilsV1.isReference(elementType);
+ AtlasAttribute attribute = ctx.getAttribute();
+ List newElements = (List) ctx.getValue();
+ AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType();
+ AtlasType elementType = arrType.getElementType();
+ List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty());
+ boolean isReference = AtlasGraphUtilsV1.isReference(elementType);
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
- List<Object> newElementsCreated = new ArrayList<>();
+ List<Object> newElementsCreated = new ArrayList<>();
if (CollectionUtils.isNotEmpty(newElements)) {
for (int index = 0; index < newElements.size(); index++) {
@@ -587,8 +704,9 @@ public class EntityGraphMapper {
if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) {
// Update the inverse reference value.
AtlasEdge newEdge = (AtlasEdge) newEntry;
- addInverseReference(arrCtx, inverseRefAttribute, newEdge);
+ addInverseReference(inverseRefAttribute, newEdge);
}
+
newElementsCreated.add(newEntry);
}
}
@@ -659,7 +777,7 @@ public class EntityGraphMapper {
case OBJECT_ID_TYPE:
AtlasEntityType instanceType = getInstanceType(ctx.getValue());
ctx.setElementType(instanceType);
- return mapObjectIdValue(ctx, context);
+ return mapObjectIdValueUsingRelationship(ctx, context);
case MAP:
case ARRAY:
@@ -816,6 +934,29 @@ public class EntityGraphMapper {
return newEdge;
}
+ private AtlasEdge updateRelationship(AtlasEdge currentEdge, final AtlasVertex entityVertex) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating entity reference using relationship {} for reference attribute {}", AtlasGraphUtilsV1.getTypeName(entityVertex));
+ }
+
+ String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentEdge.getInVertex());
+ String newEntityId = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
+ AtlasEdge ret = currentEdge;
+
+ if (!currentEntityId.equals(newEntityId) && entityVertex != null) {
+ // create a new relationship edge to the new attribute vertex from the instance
+ String relationshipName = AtlasGraphUtilsV1.getTypeName(currentEdge);
+
+ if (relationshipName == null) {
+ relationshipName = currentEdge.getLabel();
+ }
+
+ ret = createRelationship(currentEdge.getOutVertex(), entityVertex, relationshipName);
+ }
+
+ return ret;
+ }
+
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
if (AtlasGraphUtilsV1.isReference(elementType)) {
return (List)vertex.getListProperty(vertexPropertyName, AtlasEdge.class);
@@ -1038,4 +1179,26 @@ public class EntityGraphMapper {
}
}
}
+
+ private AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName) throws AtlasBaseException {
+ AtlasEdge ret = null;
+ AtlasObjectId end1 = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(end1Vertex), AtlasGraphUtilsV1.getTypeName(end1Vertex));
+ AtlasObjectId end2 = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(end2Vertex), AtlasGraphUtilsV1.getTypeName(end2Vertex));
+ AtlasRelationship relationship = relationshipStore.create(new AtlasRelationship(relationshipName, end1, end2));
+
+ // return newly created AtlasEdge
+ // if multiple edges are returned, compare using id to pick the right one
+ Iterator<AtlasEdge> outEdges = graphHelper.getOutGoingEdgesByLabel(end1Vertex, relationship.getLabel());
+
+ while (outEdges.hasNext()) {
+ AtlasEdge edge = outEdges.next();
+
+ if (AtlasGraphUtilsV1.getIdFromVertex(end2Vertex).equals(AtlasGraphUtilsV1.getIdFromVertex(edge.getInVertex()))) {
+ ret = edge;
+ break;
+ }
+ }
+
+ return ret;
+ }
}
[08/39] atlas git commit: ATLAS-1943: Fix IT failure due to incorrect
inverse reference check using relationship
Posted by ma...@apache.org.
ATLAS-1943: Fix IT failure due to incorrect inverse reference check using relationship
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/b0470f50
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/b0470f50
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/b0470f50
Branch: refs/heads/feature-odf
Commit: b0470f50e10837d052db8e0fe74847b0559bfabf
Parents: 40d909e
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Tue Jul 11 13:12:36 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 12 14:35:18 2017 -0700
----------------------------------------------------------------------
.../store/graph/AtlasRelationshipStore.java | 7 ++++++
.../graph/v1/AtlasRelationshipStoreV1.java | 25 ++++++++++++++++++++
.../store/graph/v1/EntityGraphMapper.java | 12 ++++++----
3 files changed, 39 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/b0470f50/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
index 341711a..8043760 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
@@ -46,6 +46,13 @@ public interface AtlasRelationshipStore {
AtlasRelationship getById(String guid) throws AtlasBaseException;
/**
+ * Retrieve a relationship if it exists or creates a new relationship instance.
+ * @param relationship relationship instance definition
+ * @return AtlasRelationship
+ */
+ AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException;
+
+ /**
* Delete a relationship instance using guid.
* @param guid relationship instance guid
*/
http://git-wip-us.apache.org/repos/asf/atlas/blob/b0470f50/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
index 8d9e4be..3ff6fbe 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
@@ -117,6 +117,31 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
@Override
@GraphTransaction
+ public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getOrCreate({})", relationship);
+ }
+
+ validateRelationship(relationship);
+
+ AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
+ AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
+ AtlasRelationship ret;
+
+ // check if relationship exists
+ AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship);
+
+ ret = (relationshipEdge != null) ? mapEdgeToAtlasRelationship(relationshipEdge) : create(relationship);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getOrCreate({}): {}", relationship, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
+ @GraphTransaction
public AtlasRelationship update(AtlasRelationship relationship) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> update({})", relationship);
http://git-wip-us.apache.org/repos/asf/atlas/blob/b0470f50/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index d5c1e86..68f8370 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -422,7 +422,9 @@ public class EntityGraphMapper {
AtlasEntityType entityType = (AtlasEntityType) inverseAttributeType;
if (entityType.hasRelationshipAttribute(inverseAttributeName)) {
- ret = createRelationship(inverseVertex, vertex, inverseAttribute.getRelationshipEdgeLabel());
+ String relationshipName = graphHelper.getRelationshipDefName(inverseVertex, entityType, inverseAttributeName);
+
+ ret = getOrCreateRelationship(inverseVertex, vertex, relationshipName);
} else {
if (LOG.isDebugEnabled()) {
@@ -584,7 +586,7 @@ public class EntityGraphMapper {
} else {
String relationshipName = graphHelper.getRelationshipDefName(entityVertex, entityType, attributeName);
- ret = createRelationship(entityVertex, attributeVertex, relationshipName);
+ ret = getOrCreateRelationship(entityVertex, attributeVertex, relationshipName);
}
} else {
@@ -951,7 +953,7 @@ public class EntityGraphMapper {
relationshipName = currentEdge.getLabel();
}
- ret = createRelationship(currentEdge.getOutVertex(), entityVertex, relationshipName);
+ ret = getOrCreateRelationship(currentEdge.getOutVertex(), entityVertex, relationshipName);
}
return ret;
@@ -1180,11 +1182,11 @@ public class EntityGraphMapper {
}
}
- private AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName) throws AtlasBaseException {
+ private AtlasEdge getOrCreateRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName) throws AtlasBaseException {
AtlasEdge ret = null;
AtlasObjectId end1 = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(end1Vertex), AtlasGraphUtilsV1.getTypeName(end1Vertex));
AtlasObjectId end2 = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(end2Vertex), AtlasGraphUtilsV1.getTypeName(end2Vertex));
- AtlasRelationship relationship = relationshipStore.create(new AtlasRelationship(relationshipName, end1, end2));
+ AtlasRelationship relationship = relationshipStore.getOrCreate(new AtlasRelationship(relationshipName, end1, end2));
// return newly created AtlasEdge
// if multiple edges are returned, compare using id to pick the right one
[32/39] atlas git commit: ATLAS-1979: Update storm model relationship
category and fix for UT and Coverity scan issues
Posted by ma...@apache.org.
ATLAS-1979: Update storm model relationship category and fix for UT and Coverity scan issues
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/13ba156f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/13ba156f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/13ba156f
Branch: refs/heads/feature-odf
Commit: 13ba156f1d76caaa969e2f6fd93e2387307ae7c7
Parents: e0fb7dc
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Jul 20 14:12:54 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jul 20 19:03:40 2017 -0700
----------------------------------------------------------------------
addons/models/0080-storm_model.json | 2 +-
.../repository/store/graph/v1/EntityGraphMapper.java | 9 +++++++--
.../graph/ReverseReferenceUpdateSoftDeleteTest.java | 1 +
.../store/graph/v1/AtlasRelationshipStoreV1Test.java | 10 +++++++---
4 files changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/13ba156f/addons/models/0080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json
index b008c7a..ab3d603 100644
--- a/addons/models/0080-storm_model.json
+++ b/addons/models/0080-storm_model.json
@@ -147,7 +147,7 @@
{
"name": "storm_topology_nodes",
"typeVersion": "1.0",
- "relationshipCategory": "ASSOCIATION",
+ "relationshipCategory": "AGGREGATION",
"endDef1": {
"type": "storm_topology",
"name": "nodes",
http://git-wip-us.apache.org/repos/asf/atlas/blob/13ba156f/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index 1282be5..b8fd70e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -1075,7 +1075,7 @@ public class EntityGraphMapper {
String newEntityId = getIdFromVertex(newEntityVertex);
AtlasEdge ret = currentEdge;
- if (!currentEntityId.equals(newEntityId) && newEntityVertex != null) {
+ if (!currentEntityId.equals(newEntityId)) {
// create a new relationship edge to the new attribute vertex from the instance
String relationshipName = AtlasGraphUtilsV1.getTypeName(currentEdge);
@@ -1230,6 +1230,11 @@ public class EntityGraphMapper {
// get the classification vertex from entity
String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, classification.getTypeName());
AtlasEdge classificationEdge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
+
+ if (classificationEdge == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "classificationEdge is null for label: " + relationshipLabel);
+ }
+
AtlasVertex classificationVertex = classificationEdge.getInVertex();
if (LOG.isDebugEnabled()) {
@@ -1363,7 +1368,7 @@ public class EntityGraphMapper {
private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) {
boolean ret = false;
- if (objectIds != null && objectIds.isEmpty()) {
+ if (CollectionUtils.isEmpty(objectIds)) {
ret = false;
} else {
http://git-wip-us.apache.org/repos/asf/atlas/blob/13ba156f/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
index c5eda37..ce43bdc 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/ReverseReferenceUpdateSoftDeleteTest.java
@@ -48,6 +48,7 @@ public class ReverseReferenceUpdateSoftDeleteTest extends ReverseReferenceUpdate
AtlasVertex vertexForGUID = GraphHelper.getInstance().getVertexForGUID(referencingInstance.getId()._getId());
String edgeLabel = GraphHelper.getEdgeLabel(typeB, typeB.fieldMapping.fields.get("a"));
AtlasEdge edgeForLabel = GraphHelper.getInstance().getEdgeForLabel(vertexForGUID, edgeLabel);
+ Assert.assertNotNull(edgeForLabel);
String edgeState = edgeForLabel.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
Assert.assertEquals(edgeState, Id.EntityState.DELETED.name());
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/13ba156f/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
index 31efe86..3ebda0d 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
@@ -169,7 +169,7 @@ public abstract class AtlasRelationshipStoreV1Test {
assertObjectIdEquals(johnMentorId, maxId);
List<AtlasObjectId> johnMenteesId = toAtlasObjectIds(john.getRelationshipAttribute("mentees"));
- assertNull(johnMenteesId);
+ assertEmpty(johnMenteesId);
// Jane Manager validation
AtlasObjectId janeDepartmentId = toAtlasObjectId(jane.getRelationshipAttribute("department"));
@@ -183,7 +183,7 @@ public abstract class AtlasRelationshipStoreV1Test {
assertNull(janeMentorId);
List<AtlasObjectId> janeMenteesId = toAtlasObjectIds(jane.getRelationshipAttribute("mentees"));
- assertNull(janeMenteesId);
+ assertEmpty(janeMenteesId);
List<AtlasObjectId> janeSubordinateIds = toAtlasObjectIds(jane.getRelationshipAttribute("subordinates"));
assertNotNull(janeSubordinateIds);
@@ -208,7 +208,7 @@ public abstract class AtlasRelationshipStoreV1Test {
assertObjectIdsContains(juliusMenteesId, maxId);
List<AtlasObjectId> juliusSubordinateIds = toAtlasObjectIds(julius.getRelationshipAttribute("subordinates"));
- assertNull(juliusSubordinateIds);
+ assertEmpty(juliusSubordinateIds);
}
@Test
@@ -444,6 +444,10 @@ public abstract class AtlasRelationshipStoreV1Test {
assertTrue(objId1.equals(objId2));
}
+ private static void assertEmpty(List collection) {
+ assertTrue(collection != null && collection.isEmpty());
+ }
+
private static List<AtlasObjectId> toAtlasObjectIds(Object objectIds) {
if (objectIds instanceof List) {
return (List<AtlasObjectId>) objectIds;
[12/39] atlas git commit: ATLAS-1908: updated to use existing Kafka
consumer properties when equivalent new Kafka consumer properties are not
present
Posted by ma...@apache.org.
ATLAS-1908: updated to use existing Kafka consumer properties when equivalent new Kafka consumer properties are not present
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/0d8f9f8d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/0d8f9f8d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/0d8f9f8d
Branch: refs/heads/feature-odf
Commit: 0d8f9f8d2d3902471dc246bd51d3eb7512021f74
Parents: de91608
Author: nixonrodrigues <ni...@apache.org>
Authored: Mon Jul 3 18:18:47 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 14 15:28:58 2017 -0700
----------------------------------------------------------------------
distro/src/conf/atlas-application.properties | 2 +-
.../org/apache/atlas/kafka/AtlasKafkaConsumer.java | 13 +++++++------
.../org/apache/atlas/kafka/KafkaNotification.java | 12 ++++++++++--
.../atlas/notification/NotificationConsumer.java | 3 +--
.../org/apache/atlas/kafka/KafkaConsumerTest.java | 16 ++++++++--------
.../apache/atlas/kafka/KafkaNotificationTest.java | 2 +-
.../AbstractNotificationConsumerTest.java | 8 ++++----
.../notification/NotificationHookConsumer.java | 2 +-
.../NotificationHookConsumerKafkaTest.java | 2 +-
.../atlas/web/integration/BaseResourceIT.java | 2 +-
10 files changed, 35 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index c3213df..29a4cc1 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -79,7 +79,7 @@ atlas.kafka.hook.group.id=atlas
atlas.kafka.enable.auto.commit=false
atlas.kafka.auto.offset.reset=earliest
atlas.kafka.session.timeout.ms=30000
-
+atlas.kafka.poll.timeout.ms=1000
atlas.notification.create.topics=true
atlas.notification.replicas=1
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
index 9c15243..52d0916 100644
--- a/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
+++ b/notification/src/main/java/org/apache/atlas/kafka/AtlasKafkaConsumer.java
@@ -41,19 +41,20 @@ public class AtlasKafkaConsumer<T> extends AbstractNotificationConsumer<T> {
private static final Logger LOG = LoggerFactory.getLogger(AtlasKafkaConsumer.class);
private final KafkaConsumer kafkaConsumer;
- private final boolean autoCommitEnabled;
+ private final boolean autoCommitEnabled;
+ private long pollTimeoutMilliSeconds = 1000L;
- public AtlasKafkaConsumer(MessageDeserializer<T> deserializer, KafkaConsumer kafkaConsumer, boolean autoCommitEnabled) {
+ public AtlasKafkaConsumer(MessageDeserializer<T> deserializer, KafkaConsumer kafkaConsumer, boolean autoCommitEnabled, long pollTimeoutMilliSeconds) {
super(deserializer);
-
- this.kafkaConsumer = kafkaConsumer;
+ this.kafkaConsumer = kafkaConsumer;
this.autoCommitEnabled = autoCommitEnabled;
+ this.pollTimeoutMilliSeconds = pollTimeoutMilliSeconds;
}
- public List<AtlasKafkaMessage<T>> receive(long timeoutMilliSeconds) {
+ public List<AtlasKafkaMessage<T>> receive() {
List<AtlasKafkaMessage<T>> messages = new ArrayList();
- ConsumerRecords<?, ?> records = kafkaConsumer.poll(timeoutMilliSeconds);
+ ConsumerRecords<?, ?> records = kafkaConsumer.poll(pollTimeoutMilliSeconds);
if (records != null) {
for (ConsumerRecord<?, ?> record : records) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java
index 366c8a7..38889ef 100644
--- a/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java
+++ b/notification/src/main/java/org/apache/atlas/kafka/KafkaNotification.java
@@ -83,6 +83,7 @@ public class KafkaNotification extends AbstractNotification implements Service {
private Properties properties;
private KafkaConsumer consumer = null;
private KafkaProducer producer = null;
+ private Long pollTimeOutMs = 1000L;
private static final Map<NotificationType, String> TOPIC_MAP = new HashMap<NotificationType, String>() {
{
@@ -124,6 +125,13 @@ public class KafkaNotification extends AbstractNotification implements Service {
properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.StringDeserializer");
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+
+ pollTimeOutMs = subsetConfiguration.getLong("poll.timeout.ms", 1000);
+ boolean oldApiCommitEnbleFlag = subsetConfiguration.getBoolean("auto.commit.enable",false);
+ //set old autocommit value if new autoCommit property is not set.
+ properties.put("enable.auto.commit", subsetConfiguration.getBoolean("enable.auto.commit", oldApiCommitEnbleFlag));
+ properties.put("session.timeout.ms", subsetConfiguration.getString("session.timeout.ms", "30000"));
+
}
@VisibleForTesting
@@ -167,7 +175,7 @@ public class KafkaNotification extends AbstractNotification implements Service {
public <T> List<NotificationConsumer<T>> createConsumers(NotificationType notificationType,
int numConsumers) {
return createConsumers(notificationType, numConsumers,
- Boolean.valueOf(properties.getProperty("enable.auto.commit", "true")));
+ Boolean.valueOf(properties.getProperty("enable.auto.commit", properties.getProperty("auto.commit.enable","false"))));
}
@VisibleForTesting
@@ -177,7 +185,7 @@ public class KafkaNotification extends AbstractNotification implements Service {
Properties consumerProperties = getConsumerProperties(notificationType);
List<NotificationConsumer<T>> consumers = new ArrayList<>();
- AtlasKafkaConsumer kafkaConsumer = new AtlasKafkaConsumer(notificationType.getDeserializer(), getKafkaConsumer(consumerProperties,notificationType, autoCommitEnabled), autoCommitEnabled);
+ AtlasKafkaConsumer kafkaConsumer = new AtlasKafkaConsumer(notificationType.getDeserializer(), getKafkaConsumer(consumerProperties,notificationType, autoCommitEnabled), autoCommitEnabled, pollTimeOutMs );
consumers.add(kafkaConsumer);
return consumers;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
index 22e40f9..6d1c08a 100644
--- a/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
+++ b/notification/src/main/java/org/apache/atlas/notification/NotificationConsumer.java
@@ -41,8 +41,7 @@ public interface NotificationConsumer<T> {
/**
* Fetch data for the topics from Kafka
- * @param timeoutMilliSeconds poll timeout
* @return List containing kafka message and partionId and offset.
*/
- List<AtlasKafkaMessage<T>> receive(long timeoutMilliSeconds);
+ List<AtlasKafkaMessage<T>> receive();
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java
index 70059cb..9b712f4 100644
--- a/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java
+++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaConsumerTest.java
@@ -95,12 +95,12 @@ public class KafkaConsumerTest {
ConsumerRecords records = new ConsumerRecords(mp);
- when(kafkaConsumer.poll(1000)).thenReturn(records);
+ when(kafkaConsumer.poll(100)).thenReturn(records);
when(messageAndMetadata.message()).thenReturn(json);
- AtlasKafkaConsumer consumer = new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer,false);
- List<AtlasKafkaMessage<HookNotification.HookNotificationMessage>> messageList = consumer.receive(1000);
+ AtlasKafkaConsumer consumer = new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer, false, 100L);
+ List<AtlasKafkaMessage<HookNotification.HookNotificationMessage>> messageList = consumer.receive();
assertTrue(messageList.size() > 0);
HookNotification.HookNotificationMessage consumedMessage = messageList.get(0).getMessage();
@@ -131,12 +131,12 @@ public class KafkaConsumerTest {
mp.put(tp,klist);
ConsumerRecords records = new ConsumerRecords(mp);
- when(kafkaConsumer.poll(1000)).thenReturn(records);
+ when(kafkaConsumer.poll(100L)).thenReturn(records);
when(messageAndMetadata.message()).thenReturn(json);
- AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer ,false);
+ AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer ,false, 100L);
try {
- List<AtlasKafkaMessage<HookNotification.HookNotificationMessage>> messageList = consumer.receive(1000);
+ List<AtlasKafkaMessage<HookNotification.HookNotificationMessage>> messageList = consumer.receive();
assertTrue(messageList.size() > 0);
HookNotification.HookNotificationMessage consumedMessage = messageList.get(0).getMessage();
@@ -154,7 +154,7 @@ public class KafkaConsumerTest {
TopicPartition tp = new TopicPartition("ATLAS_HOOK",0);
- AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer, false);
+ AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer, false, 100L);
consumer.commit(tp, 1);
@@ -166,7 +166,7 @@ public class KafkaConsumerTest {
TopicPartition tp = new TopicPartition("ATLAS_HOOK",0);
- AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer, true);
+ AtlasKafkaConsumer consumer =new AtlasKafkaConsumer(NotificationInterface.NotificationType.HOOK.getDeserializer(), kafkaConsumer, true , 100L);
consumer.commit(tp, 1);
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java
index c791d43..a1e13b9 100644
--- a/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java
+++ b/notification/src/test/java/org/apache/atlas/kafka/KafkaNotificationTest.java
@@ -70,7 +70,7 @@ public class KafkaNotificationTest {
List<AtlasKafkaMessage<Object>> messages = null ;
long startTime = System.currentTimeMillis(); //fetch starting time
while ((System.currentTimeMillis() - startTime) < 10000) {
- messages = consumer.receive(1000L);
+ messages = consumer.receive();
if (messages.size() > 0) {
break;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
----------------------------------------------------------------------
diff --git a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
index 8324b57..68fe3d7 100644
--- a/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
+++ b/notification/src/test/java/org/apache/atlas/notification/AbstractNotificationConsumerTest.java
@@ -67,7 +67,7 @@ public class AbstractNotificationConsumerTest {
NotificationConsumer<TestMessage> consumer =
new TestNotificationConsumer<>(versionedMessageType, jsonList, logger);
- List<AtlasKafkaMessage<TestMessage>> messageList = consumer.receive(1000L);
+ List<AtlasKafkaMessage<TestMessage>> messageList = consumer.receive();
assertFalse(messageList.isEmpty());
@@ -106,7 +106,7 @@ public class AbstractNotificationConsumerTest {
NotificationConsumer<TestMessage> consumer =
new TestNotificationConsumer<>(versionedMessageType, jsonList, logger);
- List<AtlasKafkaMessage<TestMessage>> messageList = consumer.receive(1000L);
+ List<AtlasKafkaMessage<TestMessage>> messageList = consumer.receive();
assertEquals(new TestMessage("sValue1", 99), messageList.get(0).getMessage());
@@ -138,7 +138,7 @@ public class AbstractNotificationConsumerTest {
NotificationConsumer<TestMessage> consumer =
new TestNotificationConsumer<>(versionedMessageType, jsonList, logger);
try {
- List<AtlasKafkaMessage<TestMessage>> messageList = consumer.receive(1000L);
+ List<AtlasKafkaMessage<TestMessage>> messageList = consumer.receive();
messageList.get(1).getMessage();
@@ -203,7 +203,7 @@ public class AbstractNotificationConsumerTest {
}
@Override
- public List<AtlasKafkaMessage<T>> receive(long timeoutMilliSeconds) {
+ public List<AtlasKafkaMessage<T>> receive() {
List<AtlasKafkaMessage<T>> tempMessageList = new ArrayList();
for(Object json : messageList) {
tempMessageList.add(new AtlasKafkaMessage(deserializer.deserialize((String)json), -1, -1));
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
index 0dea0e2..51276d3 100644
--- a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
+++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
@@ -224,7 +224,7 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
while (shouldRun.get()) {
try {
- List<AtlasKafkaMessage<HookNotificationMessage>> messages = consumer.receive(1000L);
+ List<AtlasKafkaMessage<HookNotificationMessage>> messages = consumer.receive();
for (AtlasKafkaMessage<HookNotificationMessage> msg : messages) {
handleMessage(msg);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java
index 650ca0a..eb37fa8 100644
--- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java
+++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerKafkaTest.java
@@ -155,7 +155,7 @@ public class NotificationHookConsumerKafkaTest {
try {
long startTime = System.currentTimeMillis(); //fetch starting time
while ((System.currentTimeMillis() - startTime) < 10000) {
- List<AtlasKafkaMessage<HookNotificationMessage>> messages = consumer.receive(1000L);
+ List<AtlasKafkaMessage<HookNotificationMessage>> messages = consumer.receive();
for (AtlasKafkaMessage<HookNotificationMessage> msg : messages) {
hookConsumer.handleMessage(msg);
http://git-wip-us.apache.org/repos/asf/atlas/blob/0d8f9f8d/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
index c036cfa..496185f 100755
--- a/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/integration/BaseResourceIT.java
@@ -637,7 +637,7 @@ public abstract class BaseResourceIT {
try {
while (System.currentTimeMillis() < maxCurrentTime) {
- List<AtlasKafkaMessage<EntityNotification>> messageList = consumer.receive(1000);
+ List<AtlasKafkaMessage<EntityNotification>> messageList = consumer.receive();
if(messageList.size() > 0) {
EntityNotification notification = messageList.get(0).getMessage();
if (predicate.evaluate(notification)) {
[13/39] atlas git commit: ATLAS-1947: AtlasSearchResult to include
referredEntity headers
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java b/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java
deleted file mode 100644
index 0f91b2d..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchPipeline.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasStructType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.util.SearchTracker;
-import org.apache.atlas.utils.AtlasPerfTracer;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.builder.ToStringBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-@Component
-public class SearchPipeline {
- private static final Logger LOG = LoggerFactory.getLogger(SearchPipeline.class);
- private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("SearchPipeline");
-
- enum ExecutionMode { SOLR, GREMLIN, MIXED }
-
- enum IndexResultType { TAG, ENTITY, TEXT }
-
- private final SolrStep solrStep;
- private final GremlinStep gremlinStep;
- private final SearchTracker searchTracker;
- private final AtlasTypeRegistry typeRegistry;
- private final Configuration atlasConfiguration;
- private final GraphBackedSearchIndexer indexer;
-
- @Inject
- public SearchPipeline(SolrStep solrStep, GremlinStep gremlinStep, SearchTracker searchTracker, AtlasTypeRegistry typeRegistry, Configuration atlasConfiguration, GraphBackedSearchIndexer indexer) {
- this.solrStep = solrStep;
- this.gremlinStep = gremlinStep;
- this.searchTracker = searchTracker;
- this.typeRegistry = typeRegistry;
- this.atlasConfiguration = atlasConfiguration;
- this.indexer = indexer;
- }
-
- public List<AtlasVertex> run(SearchParameters searchParameters) throws AtlasBaseException {
- final List<AtlasVertex> ret;
-
- AtlasPerfTracer perf = null;
-
- if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "SearchPipeline.run("+ searchParameters +")");
- }
-
- AtlasEntityType entityType = typeRegistry.getEntityTypeByName(searchParameters.getTypeName());
- AtlasClassificationType classiType = typeRegistry.getClassificationTypeByName(searchParameters.getClassification());
- PipelineContext context = new PipelineContext(searchParameters, entityType, classiType, indexer.getVertexIndexKeys());
- String searchId = searchTracker.add(context); // For future cancellation
-
- try {
- ExecutionMode mode = determineExecutionMode(context);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Execution mode {}", mode);
- }
-
- switch (mode) {
- case SOLR:
- ret = runOnlySolr(context);
- break;
-
- case GREMLIN:
- ret = runOnlyGremlin(context);
- break;
-
- case MIXED:
- ret = runMixed(context);
- break;
-
- default:
- ret = Collections.emptyList();
- }
- } finally {
- searchTracker.remove(searchId);
-
- AtlasPerfTracer.log(perf);
- }
-
- return ret;
- }
-
- private List<AtlasVertex> runOnlySolr(PipelineContext context) throws AtlasBaseException {
- // Only when there's no tag and query
- List<AtlasVertex> results = new ArrayList<>();
-
- while (results.size() < context.getSearchParameters().getLimit()) {
- if (context.getForceTerminate()) {
- LOG.debug("search has been terminated");
-
- break;
- }
-
- // Execute solr search only
- solrStep.execute(context);
-
- List<AtlasVertex> stepResults = getIndexResults(context);
-
- context.incrementSearchRound();
-
- addToResult(results, stepResults, context.getSearchParameters().getLimit());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Pipeline iteration {}: stepResults={}; totalResult={}", context.getIterationCount(), stepResults.size(), results.size());
- }
-
- if (CollectionUtils.isEmpty(stepResults)) {
- // If no result is found any subsequent iteration then just stop querying the index
- break;
- }
- }
-
- if (context.getIndexResultType() == IndexResultType.TAG) {
- List<AtlasVertex> entityVertices = new ArrayList<>(results.size());
-
- for (AtlasVertex tagVertex : results) {
- Iterable<AtlasEdge> edges = tagVertex.getEdges(AtlasEdgeDirection.IN);
-
- for (AtlasEdge edge : edges) {
- AtlasVertex entityVertex = edge.getOutVertex();
-
- entityVertices.add(entityVertex);
- }
- }
-
- results = entityVertices;
- }
-
- return results;
- }
-
- private List<AtlasVertex> runOnlyGremlin(PipelineContext context) throws AtlasBaseException {
- List<AtlasVertex> results = new ArrayList<>();
-
- while (results.size() < context.getSearchParameters().getLimit()) {
- if (context.getForceTerminate()) {
- LOG.debug("search has been terminated");
-
- break;
- }
-
- gremlinStep.execute(context);
-
- List<AtlasVertex> stepResults = getGremlinResults(context);
-
- context.incrementSearchRound();
-
- addToResult(results, stepResults, context.getSearchParameters().getLimit());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Pipeline iteration {}: stepResults={}; totalResult={}", context.getIterationCount(), stepResults.size(), results.size());
- }
-
- if (CollectionUtils.isEmpty(stepResults)) {
- // If no result is found any subsequent iteration then just stop querying the index
- break;
- }
- }
-
- return results;
- }
-
- /*
- 1. Index processes few attributes and then gremlin processes rest
- 1.1 Iterate for gremlin till the index results are non null
- 2. Index processes all attributes, gremlin has nothing to do
-
- Sometimes the result set might be less than the max limit and we need to iterate until the result set is full
- or the iteration doesn't return any results
-
- */
- private List<AtlasVertex> runMixed(PipelineContext context) throws AtlasBaseException {
- List<AtlasVertex> results = new ArrayList<>();
-
- while (results.size() < context.getSearchParameters().getLimit()) {
- if (context.getForceTerminate()) {
- LOG.debug("search has been terminated");
-
- break;
- }
-
- // Execute Solr search and then pass it to the Gremlin step (if needed)
- solrStep.execute(context);
-
- if (!context.hasIndexResults()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No index results in iteration {}", context.getIterationCount());
- }
-
- // If no result is found any subsequent iteration then just stop querying the index
- break;
- }
-
- // Attributes partially processed by Solr, use gremlin to process remaining attribute(s)
- gremlinStep.execute(context);
-
- context.incrementSearchRound();
-
- List<AtlasVertex> stepResults = getGremlinResults(context);
-
- addToResult(results, stepResults, context.getSearchParameters().getLimit());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Pipeline iteration {}: stepResults={}; totalResult={}", context.getIterationCount(), stepResults.size(), results.size());
- }
- }
-
- return results;
- }
-
- private void addToResult(List<AtlasVertex> result, List<AtlasVertex> stepResult, int maxLimit) {
- if (result != null && stepResult != null && result.size() < maxLimit) {
- for (AtlasVertex vertex : stepResult) {
- result.add(vertex);
-
- if (result.size() >= maxLimit) {
- break;
- }
- }
- }
- }
-
- private List<AtlasVertex> getIndexResults(PipelineContext pipelineContext) {
- List<AtlasVertex> ret = new ArrayList<>();
-
- if (pipelineContext.hasIndexResults()) {
- Iterator<AtlasIndexQuery.Result> iter = pipelineContext.getIndexResultsIterator();
-
- while(iter.hasNext()) {
- ret.add(iter.next().getVertex());
- }
- }
-
- return ret;
- }
-
- private List<AtlasVertex> getGremlinResults(PipelineContext pipelineContext) {
- List<AtlasVertex> ret = new ArrayList<>();
-
- if (pipelineContext.hasGremlinResults()) {
- Iterator<AtlasVertex> iter = pipelineContext.getGremlinResultIterator();
-
- while (iter.hasNext()) {
- ret.add(iter.next());
- }
- }
-
- return ret;
- }
-
- private ExecutionMode determineExecutionMode(PipelineContext context) {
- SearchParameters searchParameters = context.getSearchParameters();
- AtlasClassificationType classificationType = context.getClassificationType();
- AtlasEntityType entityType = context.getEntityType();
- int solrCount = 0;
- int gremlinCount = 0;
-
- if (StringUtils.isNotEmpty(searchParameters.getQuery())) {
- solrCount++;
-
- // __state index only exists in vertex_index
- if (searchParameters.getExcludeDeletedEntities()) {
- gremlinCount++;
- }
- }
-
- if (classificationType != null) {
- Set<String> typeAndAllSubTypes = classificationType.getTypeAndAllSubTypes();
-
- if (typeAndAllSubTypes.size() > atlasConfiguration.getInt(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Classification type {} has too many subTypes ({}) to use in Solr search. Gremlin will be used to execute the search",
- classificationType.getTypeName(), typeAndAllSubTypes.size());
- }
-
- gremlinCount++;
- } else {
- if (hasNonIndexedAttrViolation(classificationType, context.getIndexedKeys(), searchParameters.getTagFilters())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Tag filters not suitable for Solr search. Gremlin will be used to execute the search");
- }
-
- gremlinCount++;
- } else {
- solrCount++;
-
- // __state index only exist in vertex_index
- if (searchParameters.getExcludeDeletedEntities()) {
- gremlinCount++;
- }
- }
- }
- }
-
- if (entityType != null) {
- Set<String> typeAndAllSubTypes = entityType.getTypeAndAllSubTypes();
-
- if (typeAndAllSubTypes.size() > atlasConfiguration.getInt(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Entity type {} has too many subTypes ({}) to use in Solr search. Gremlin will be used to execute the search",
- entityType.getTypeName(), typeAndAllSubTypes.size());
- }
-
- gremlinCount++;
- } else {
- if (hasNonIndexedAttrViolation(entityType, context.getIndexedKeys(), searchParameters.getEntityFilters())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Entity filters not suitable for Solr search. Gremlin will be used to execute the search");
- }
-
- gremlinCount++;
- } else {
- solrCount++;
- }
- }
- }
-
- ExecutionMode mode = ExecutionMode.MIXED;
-
- if (solrCount == 1 && gremlinCount == 0) {
- mode = ExecutionMode.SOLR;
- } else if (gremlinCount == 1 && solrCount == 0) {
- mode = ExecutionMode.GREMLIN;
- }
-
- return mode;
- }
-
- // If Index can't process all attributes and any of the non-indexed attribute is present in OR nested within AND
- // then the only way is Gremlin
- // A violation (here) is defined as presence of non-indexed attribute within any OR clause nested under an AND clause
- // the reason being that the index would not be able to process the nested OR attribute which might result in
- // exclusion of valid result (vertex)
- private boolean hasNonIndexedAttrViolation(AtlasStructType structType, Set<String> indexKeys, FilterCriteria filterCriteria) {
- return hasNonIndexedAttrViolation(structType, indexKeys, filterCriteria, false);
- }
-
- private boolean hasNonIndexedAttrViolation(AtlasStructType structType, Set<String> indexKeys, FilterCriteria filterCriteria, boolean enclosedInOrCondition) {
- if (filterCriteria == null) {
- return false;
- }
-
- boolean ret = false;
- Condition filterCondition = filterCriteria.getCondition();
- List<FilterCriteria> criterion = filterCriteria.getCriterion();
-
- if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
- if (!enclosedInOrCondition) {
- enclosedInOrCondition = filterCondition == Condition.OR;
- }
-
- // If we have nested criterion let's find any nested ORs with non-indexed attr
- for (FilterCriteria criteria : criterion) {
- ret |= hasNonIndexedAttrViolation(structType, indexKeys, criteria, enclosedInOrCondition);
-
- if (ret) {
- break;
- }
- }
- } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
- // If attribute qualified name doesn't exist in the vertex index we potentially might have a problem
- try {
- String qualifiedAttributeName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
-
- ret = CollectionUtils.isEmpty(indexKeys) || !indexKeys.contains(qualifiedAttributeName);
-
- if (ret) {
- LOG.warn("search includes non-indexed attribute '{}'; might cause poor performance", qualifiedAttributeName);
- }
- } catch (AtlasBaseException e) {
- LOG.warn(e.getMessage());
-
- ret = true;
- }
- }
-
- // return ret && enclosedInOrCondition;
-
- return ret;
- }
-
- public interface PipelineStep {
- void execute(PipelineContext context) throws AtlasBaseException;
- }
-
- public static class PipelineContext {
- // TODO: See if anything can be cached in the context
-
- private final SearchParameters searchParameters;
- private final AtlasEntityType entityType;
- private final AtlasClassificationType classificationType;
- private final Set<String> indexedKeys;
-
- private int iterationCount;
- private boolean forceTerminate;
- private int currentOffset;
- private int maxLimit;
-
- // Continuous processing stuff
- private Set<String> tagSearchAttributes = new HashSet<>();
- private Set<String> entitySearchAttributes = new HashSet<>();
- private Set<String> tagAttrProcessedBySolr = new HashSet<>();
- private Set<String> entityAttrProcessedBySolr = new HashSet<>();
-
- // Results related stuff
- private IndexResultType indexResultType;
- private Iterator<AtlasIndexQuery.Result> indexResultsIterator;
- private Iterator<AtlasVertex> gremlinResultIterator;
-
- private Map<String, AtlasIndexQuery> cachedIndexQueries = new HashMap<>();
- private Map<String, AtlasGraphQuery> cachedGraphQueries = new HashMap<>();
-
- public PipelineContext(SearchParameters searchParameters, AtlasEntityType entityType, AtlasClassificationType classificationType, Set<String> indexedKeys) {
- this.searchParameters = searchParameters;
- this.entityType = entityType;
- this.classificationType = classificationType;
- this.indexedKeys = indexedKeys;
-
- currentOffset = searchParameters.getOffset();
- maxLimit = searchParameters.getLimit();
- }
-
- public SearchParameters getSearchParameters() {
- return searchParameters;
- }
-
- public AtlasEntityType getEntityType() {
- return entityType;
- }
-
- public AtlasClassificationType getClassificationType() {
- return classificationType;
- }
-
- public Set<String> getIndexedKeys() { return indexedKeys; }
-
- public int getIterationCount() {
- return iterationCount;
- }
-
- public boolean getForceTerminate() {
- return forceTerminate;
- }
-
- public void setForceTerminate(boolean forceTerminate) {
- this.forceTerminate = forceTerminate;
- }
-
- public boolean hasProcessedTagAttribute(String attributeName) {
- return tagAttrProcessedBySolr.contains(attributeName);
- }
-
- public boolean hasProcessedEntityAttribute(String attributeName) {
- return entityAttrProcessedBySolr.contains(attributeName);
- }
-
- public Iterator<AtlasIndexQuery.Result> getIndexResultsIterator() {
- return indexResultsIterator;
- }
-
- public void setIndexResultsIterator(Iterator<AtlasIndexQuery.Result> indexResultsIterator) {
- this.indexResultsIterator = indexResultsIterator;
- }
-
- public Iterator<AtlasVertex> getGremlinResultIterator() {
- return gremlinResultIterator;
- }
-
- public void setGremlinResultIterator(Iterator<AtlasVertex> gremlinResultIterator) {
- this.gremlinResultIterator = gremlinResultIterator;
- }
-
- public boolean hasIndexResults() {
- return null != indexResultsIterator && indexResultsIterator.hasNext();
- }
-
- public boolean hasGremlinResults() {
- return null != gremlinResultIterator && gremlinResultIterator.hasNext();
- }
-
-
- public boolean isTagProcessingComplete() {
- return CollectionUtils.isEmpty(tagSearchAttributes) ||
- CollectionUtils.isEqualCollection(tagSearchAttributes, tagAttrProcessedBySolr);
- }
-
- public boolean isEntityProcessingComplete() {
- return CollectionUtils.isEmpty(entitySearchAttributes) ||
- CollectionUtils.isEqualCollection(entitySearchAttributes, entityAttrProcessedBySolr);
- }
-
- public boolean isProcessingComplete() {
- return isTagProcessingComplete() && isEntityProcessingComplete();
- }
-
- public void incrementOffset(int increment) {
- currentOffset += increment;
- }
-
- public void incrementSearchRound() {
- iterationCount ++;
- incrementOffset(searchParameters.getLimit());
- }
-
- public int getCurrentOffset() {
- return currentOffset;
- }
-
- public boolean addTagSearchAttribute(String attribute) {
- return tagSearchAttributes.add(attribute);
- }
-
- public boolean addProcessedTagAttribute(String attribute) {
- return tagAttrProcessedBySolr.add(attribute);
- }
-
- public boolean addEntitySearchAttribute(String attribute) {
- return tagSearchAttributes.add(attribute);
- }
-
- public boolean addProcessedEntityAttribute(String attribute) {
- return entityAttrProcessedBySolr.add(attribute);
- }
-
- public void cacheGraphQuery(String name, AtlasGraphQuery graphQuery) {
- cachedGraphQueries.put(name, graphQuery);
- }
-
- public void cacheIndexQuery(String name, AtlasIndexQuery indexQuery) {
- cachedIndexQueries.put(name, indexQuery);
- }
-
- public AtlasIndexQuery getIndexQuery(String name){
- return cachedIndexQueries.get(name);
- }
-
- public AtlasGraphQuery getGraphQuery(String name) {
- return cachedGraphQueries.get(name);
- }
-
- public IndexResultType getIndexResultType() {
- return indexResultType;
- }
-
- public void setIndexResultType(IndexResultType indexResultType) {
- this.indexResultType = indexResultType;
- }
-
- public int getMaxLimit() {
- return maxLimit;
- }
-
- @Override
- public String toString() {
- return new ToStringBuilder(this)
- .append("iterationCount", iterationCount)
- .append("forceTerminate", forceTerminate)
- .append("currentOffset", currentOffset)
- .append("maxLimit", maxLimit)
- .append("searchParameters", searchParameters)
- .append("tagSearchAttributes", tagSearchAttributes)
- .append("entitySearchAttributes", entitySearchAttributes)
- .append("tagAttrProcessedBySolr", tagAttrProcessedBySolr)
- .append("entityAttrProcessedBySolr", entityAttrProcessedBySolr)
- .append("indexResultType", indexResultType)
- .append("cachedIndexQueries", cachedIndexQueries)
- .append("cachedGraphQueries", cachedGraphQueries)
- .toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
new file mode 100644
index 0000000..1a2d997
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -0,0 +1,381 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+import java.util.regex.Pattern;
+
+public abstract class SearchProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class);
+
+ public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
+ public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
+ public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
+ public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
+ public static final int MAX_ENTITY_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TYPES_COUNT, 10);
+ public static final int MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY = getApplicationProperty(Constants.INDEX_SEARCH_MAX_TAGS_COUNT, 10);
+ public static final String AND_STR = " AND ";
+ public static final String EMPTY_STRING = "";
+ public static final String SPACE_STRING = " ";
+ public static final String BRACE_OPEN_STR = "( ";
+ public static final String BRACE_CLOSE_STR = " )";
+
+ private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
+
+ static
+ {
+ OPERATOR_MAP.put(SearchParameters.Operator.LT,"v.\"%s\": [* TO %s}");
+ OPERATOR_MAP.put(SearchParameters.Operator.GT,"v.\"%s\": {%s TO *]");
+ OPERATOR_MAP.put(SearchParameters.Operator.LTE,"v.\"%s\": [* TO %s]");
+ OPERATOR_MAP.put(SearchParameters.Operator.GTE,"v.\"%s\": [%s TO *]");
+ OPERATOR_MAP.put(SearchParameters.Operator.EQ,"v.\"%s\": %s");
+ OPERATOR_MAP.put(SearchParameters.Operator.NEQ,"v.\"%s\": (NOT %s)");
+ OPERATOR_MAP.put(SearchParameters.Operator.IN, "v.\"%s\": (%s)");
+ OPERATOR_MAP.put(SearchParameters.Operator.LIKE, "v.\"%s\": (%s)");
+ OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, "v.\"%s\": (%s*)");
+ OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, "v.\"%s\": (*%s)");
+ OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, "v.\"%s\": (*%s*)");
+ }
+
+ protected final SearchContext context;
+ protected SearchProcessor nextProcessor;
+
+
+ protected SearchProcessor(SearchContext context) {
+ this.context = context;
+ }
+
+ public void addProcessor(SearchProcessor processor) {
+ if (nextProcessor == null) {
+ nextProcessor = processor;
+ } else {
+ nextProcessor.addProcessor(processor);
+ }
+ }
+
+ public abstract List<AtlasVertex> execute();
+
+ public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
+ return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices);
+ }
+
+
+ protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> solrFiltered, Set<String> gremlinFiltered, Set<String> allAttributes) {
+ if (structType == null || filterCriteria == null) {
+ return;
+ }
+
+ Condition filterCondition = filterCriteria.getCondition();
+ List<FilterCriteria> criterion = filterCriteria.getCriterion();
+
+ if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
+ for (SearchParameters.FilterCriteria criteria : criterion) {
+ processSearchAttributes(structType, criteria, solrFiltered, gremlinFiltered, allAttributes);
+ }
+ } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
+ try {
+ String attributeName = filterCriteria.getAttributeName();
+ String qualifiedName = structType.getQualifiedAttributeName(attributeName);
+ Set<String> indexedKeys = context.getIndexedKeys();
+
+ if (indexedKeys != null && indexedKeys.contains(qualifiedName)) {
+ solrFiltered.add(attributeName);
+ } else {
+ LOG.warn("search includes non-indexed attribute '{}'; might cause poor performance", qualifiedName);
+
+ gremlinFiltered.add(attributeName);
+ }
+
+ allAttributes.add(attributeName);
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
+ }
+ }
+
+ //
+ // If filterCriteria contains any non-indexed attribute inside OR condition:
+ // Solr+Grelin can't be used. Need to use only Gremlin filter for all attributes. Examples:
+ // (OR idx-att1=x non-idx-attr=z)
+ // (AND idx-att1=x (OR idx-attr2=y non-idx-attr=z))
+ // Else
+ // Solr can be used for indexed-attribute filtering and Gremlin for non-indexed attributes. Examples:
+ // (AND idx-att1=x idx-attr2=y non-idx-attr=z)
+ // (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z)
+ // (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc))
+ //
+ protected boolean canApplySolrFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) {
+ if (filterCriteria == null) {
+ return true;
+ }
+
+ boolean ret = true;
+ Condition filterCondition = filterCriteria.getCondition();
+ List<FilterCriteria> criterion = filterCriteria.getCriterion();
+ Set<String> indexedKeys = context.getIndexedKeys();
+
+
+ if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
+ insideOrCondition = insideOrCondition || filterCondition == Condition.OR;
+
+ // If we have nested criterion let's find any nested ORs with non-indexed attr
+ for (FilterCriteria criteria : criterion) {
+ ret = canApplySolrFilter(structType, criteria, insideOrCondition);
+
+ if (!ret) {
+ break;
+ }
+ }
+ } else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
+ try {
+ String qualifiedName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
+
+ if (insideOrCondition && (indexedKeys == null || !indexedKeys.contains(qualifiedName))) {
+ ret = false;
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
+ }
+
+ return ret;
+ }
+
+ protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
+ String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
+
+ solrQuery.append("v.\"__typeName\": (")
+ .append(typeAndSubtypesString)
+ .append(")");
+ }
+
+ protected void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> solrAttributes) {
+ if (filterCriteria != null) {
+ LOG.debug("Processing Filters");
+
+ String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes);
+
+ if (StringUtils.isNotEmpty(filterQuery)) {
+ solrQuery.append(AND_STR).append(filterQuery);
+ }
+ }
+
+ if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
+ solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE");
+ }
+ }
+
+ private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes) {
+ return toSolrQuery(type, criteria, solrAttributes, new StringBuilder());
+ }
+
+ private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, StringBuilder sb) {
+ if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
+ StringBuilder nestedExpression = new StringBuilder();
+
+ for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+ String nestedQuery = toSolrQuery(type, filterCriteria, solrAttributes);
+
+ if (StringUtils.isNotEmpty(nestedQuery)) {
+ if (nestedExpression.length() > 0) {
+ nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING);
+ }
+
+ nestedExpression.append(nestedQuery);
+ }
+ }
+
+ return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression.toString()).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
+ } else if (solrAttributes.contains(criteria.getAttributeName())){
+ return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
+ } else {
+ return EMPTY_STRING;
+ }
+ }
+
+ private String toSolrExpression(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) {
+ String ret = EMPTY_STRING;
+
+ try {
+ String qualifiedName = type.getQualifiedAttributeName(attrName);
+
+ if (OPERATOR_MAP.get(op) != null) {
+ ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal);
+ }
+ } catch (AtlasBaseException ex) {
+ LOG.warn(ex.getMessage());
+ }
+
+ return ret;
+ }
+
+ protected AtlasGraphQuery toGremlinFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> gremlinAttributes, AtlasGraphQuery query) {
+ if (criteria != null) {
+ if (criteria.getCondition() != null) {
+ if (criteria.getCondition() == Condition.AND) {
+ for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+ AtlasGraphQuery nestedQuery = toGremlinFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
+
+ query.addConditionsFrom(nestedQuery);
+ }
+ } else {
+ List<AtlasGraphQuery> orConditions = new LinkedList<>();
+
+ for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+ AtlasGraphQuery nestedQuery = toGremlinFilterQuery(type, filterCriteria, gremlinAttributes, context.getGraph().query());
+
+ orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
+ }
+
+ if (!orConditions.isEmpty()) {
+ query.or(orConditions);
+ }
+ }
+ } else if (gremlinAttributes.contains(criteria.getAttributeName())) {
+ String attrName = criteria.getAttributeName();
+ String attrValue = criteria.getAttributeValue();
+ SearchParameters.Operator operator = criteria.getOperator();
+
+ try {
+ final String qualifiedName = type.getQualifiedAttributeName(attrName);
+
+ switch (operator) {
+ case LT:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
+ break;
+ case LTE:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
+ break;
+ case GT:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
+ break;
+ case GTE:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
+ break;
+ case EQ:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
+ break;
+ case NEQ:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
+ break;
+ case LIKE:
+ // TODO: Maybe we need to validate pattern
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getLikeRegex(attrValue));
+ break;
+ case CONTAINS:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
+ break;
+ case STARTS_WITH:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
+ break;
+ case ENDS_WITH:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
+ break;
+ case IN:
+ LOG.warn("{}: unsupported operator. Ignored", operator);
+ break;
+ }
+ } catch (AtlasBaseException e) {
+ LOG.error("toGremlinFilterQuery(): failed for attrName=" + attrName + "; operator=" + operator + "; attrValue=" + attrValue, e);
+ }
+ }
+ }
+
+ return query;
+ }
+
+ private String getContainsRegex(String attributeValue) {
+ return ".*" + attributeValue + ".*";
+ }
+
+ private String getSuffixRegex(String attributeValue) {
+ return ".*" + attributeValue;
+ }
+
+ private String getLikeRegex(String attributeValue) { return ".*" + attributeValue + ".*"; }
+
+ protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult) {
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ if (idxQueryResult != null) {
+ while (idxQueryResult.hasNext()) {
+ AtlasVertex vertex = idxQueryResult.next().getVertex();
+
+ ret.add(vertex);
+ }
+ }
+
+ return ret;
+ }
+
+ protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> vertices) {
+ List<AtlasVertex> ret = new ArrayList<>();
+
+ if (vertices != null) {
+ while (vertices.hasNext()) {
+ AtlasVertex vertex = vertices.next();
+
+ ret.add(vertex);
+ }
+ }
+
+ return ret;
+ }
+
+ protected Set<String> getGuids(List<AtlasVertex> vertices) {
+ Set<String> ret = new HashSet<>();
+
+ if (vertices != null) {
+ for(AtlasVertex vertex : vertices) {
+ String guid = AtlasGraphUtilsV1.getIdFromVertex(vertex);
+
+ if (StringUtils.isNotEmpty(guid)) {
+ ret.add(guid);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ private static int getApplicationProperty(String propertyName, int defaultValue) {
+ try {
+ return ApplicationProperties.get().getInt(propertyName, defaultValue);
+ } catch (AtlasException excp) {
+ // ignore
+ }
+
+ return defaultValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java b/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java
deleted file mode 100644
index 6a5dd5a..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/SolrStep.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.discovery.SearchPipeline.IndexResultType;
-import org.apache.atlas.discovery.SearchPipeline.PipelineContext;
-import org.apache.atlas.discovery.SearchPipeline.PipelineStep;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.Operator;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
-import org.apache.atlas.type.*;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import static org.apache.atlas.model.discovery.SearchParameters.*;
-
-@Component
-public class SolrStep implements PipelineStep {
- private static final Logger LOG = LoggerFactory.getLogger(SolrStep.class);
-
- private static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
- private static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
- private static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
- private static final String AND_STR = " AND ";
- private static final String EMPTY_STRING = "";
- private static final String SPACE_STRING = " ";
- private static final String BRACE_OPEN_STR = "( ";
- private static final String BRACE_CLOSE_STR = " )";
-
- private static final Map<Operator, String> operatorMap = new HashMap<>();
-
- static
- {
- operatorMap.put(Operator.LT,"v.\"%s\": [* TO %s}");
- operatorMap.put(Operator.GT,"v.\"%s\": {%s TO *]");
- operatorMap.put(Operator.LTE,"v.\"%s\": [* TO %s]");
- operatorMap.put(Operator.GTE,"v.\"%s\": [%s TO *]");
- operatorMap.put(Operator.EQ,"v.\"%s\": %s");
- operatorMap.put(Operator.NEQ,"v.\"%s\": (NOT %s)");
- operatorMap.put(Operator.IN, "v.\"%s\": (%s)");
- operatorMap.put(Operator.LIKE, "v.\"%s\": (%s)");
- operatorMap.put(Operator.STARTS_WITH, "v.\"%s\": (%s*)");
- operatorMap.put(Operator.ENDS_WITH, "v.\"%s\": (*%s)");
- operatorMap.put(Operator.CONTAINS, "v.\"%s\": (*%s*)");
- }
-
- private final AtlasGraph graph;
-
- @Inject
- public SolrStep(AtlasGraph graph) {
- this.graph = graph;
- }
-
- @Override
- public void execute(PipelineContext context) throws AtlasBaseException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> SolrStep.execute({})", context);
- }
-
- if (context == null) {
- throw new AtlasBaseException("Can't start search without any context");
- }
-
- SearchParameters searchParameters = context.getSearchParameters();
-
- final Iterator<AtlasIndexQuery.Result> result;
-
- if (StringUtils.isNotEmpty(searchParameters.getQuery())) {
- result = executeAgainstFulltextIndex(context);
- } else {
- result = executeAgainstVertexIndex(context);
- }
-
- context.setIndexResultsIterator(result);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== SolrStep.execute({})", context);
- }
- }
-
- private Iterator<AtlasIndexQuery.Result> executeAgainstFulltextIndex(PipelineContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> SolrStep.executeAgainstFulltextIndex()");
- }
-
- final Iterator<AtlasIndexQuery.Result> ret;
-
- AtlasIndexQuery query = context.getIndexQuery("FULLTEXT");
-
- if (query == null) {
- // Compute only once
- SearchParameters searchParameters = context.getSearchParameters();
- String indexQuery = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
-
- query = graph.indexQuery(Constants.FULLTEXT_INDEX, indexQuery);
-
- context.cacheIndexQuery("FULLTEXT", query);
- }
-
- context.setIndexResultType(IndexResultType.TEXT);
-
- ret = query.vertices(context.getCurrentOffset(), context.getMaxLimit());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== SolrStep.executeAgainstFulltextIndex()");
- }
-
- return ret;
- }
-
- private Iterator<AtlasIndexQuery.Result> executeAgainstVertexIndex(PipelineContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> SolrStep.executeAgainstVertexIndex()");
- }
-
- final Iterator<AtlasIndexQuery.Result> ret;
-
- SearchParameters searchParameters = context.getSearchParameters();
- AtlasIndexQuery query = context.getIndexQuery("VERTEX_INDEX");
-
- if (query == null) {
- StringBuilder solrQuery = new StringBuilder();
-
- // If tag is specified then let's start processing using tag and it's attributes, entity filters will
- // be pushed to Gremlin
- if (context.getClassificationType() != null) {
- context.setIndexResultType(IndexResultType.TAG);
-
- constructTypeTestQuery(solrQuery, context.getClassificationType().getTypeAndAllSubTypes());
- constructFilterQuery(solrQuery, context.getClassificationType(), searchParameters.getTagFilters(), context);
- } else if (context.getEntityType() != null) {
- context.setIndexResultType(IndexResultType.ENTITY);
-
- constructTypeTestQuery(solrQuery, context.getEntityType().getTypeAndAllSubTypes());
- constructFilterQuery(solrQuery, context.getEntityType(), searchParameters.getEntityFilters(), context);
-
- // Set the status flag
- if (searchParameters.getExcludeDeletedEntities()) {
- if (solrQuery.length() > 0) {
- solrQuery.append(" AND ");
- }
-
- solrQuery.append("v.\"__state\":").append("ACTIVE");
- }
- }
-
- // No query was formed, doesn't make sense to do anything beyond this point
- if (solrQuery.length() > 0) {
- String validSolrQuery = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
- validSolrQuery = STRAY_OR_PATTERN.matcher(validSolrQuery).replaceAll(")");
- validSolrQuery = STRAY_ELIPSIS_PATTERN.matcher(validSolrQuery).replaceAll(EMPTY_STRING);
-
- query = graph.indexQuery(Constants.VERTEX_INDEX, validSolrQuery);
- context.cacheIndexQuery("VERTEX_INDEX", query);
- }
- }
-
- // Execute solr query and return the index results in the context
- if (query != null) {
- ret = query.vertices(context.getCurrentOffset(), context.getMaxLimit());
- } else {
- ret = null;
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== SolrStep.executeAgainstVertexIndex()");
- }
-
- return ret;
- }
-
- private void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
- String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
-
- solrQuery.append("v.\"__typeName\": (")
- .append(typeAndSubtypesString)
- .append(")");
- }
-
- private void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, PipelineContext context) {
- if (filterCriteria != null) {
- LOG.debug("Processing Filters");
-
- String filterQuery = toSolrQuery(type, filterCriteria, context);
-
- if (StringUtils.isNotEmpty(filterQuery)) {
- solrQuery.append(AND_STR).append(filterQuery);
- }
- }
- }
-
- private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, PipelineContext context) {
- return toSolrQuery(type, criteria, context, new StringBuilder());
- }
-
- private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, PipelineContext context, StringBuilder sb) {
- if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
- StringBuilder nestedExpression = new StringBuilder();
-
- for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- String nestedQuery = toSolrQuery(type, filterCriteria, context);
-
- if (StringUtils.isNotEmpty(nestedQuery)) {
- if (nestedExpression.length() > 0) {
- nestedExpression.append(SPACE_STRING).append(criteria.getCondition()).append(SPACE_STRING);
- }
-
- nestedExpression.append(nestedQuery);
- }
- }
-
- return nestedExpression.length() > 0 ? sb.append(BRACE_OPEN_STR).append(nestedExpression.toString()).append(BRACE_CLOSE_STR).toString() : EMPTY_STRING;
- } else {
- return toSolrExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue(), context);
- }
- }
-
- private String toSolrExpression(AtlasStructType type, String attrName, Operator op, String attrVal, PipelineContext context) {
- String ret = EMPTY_STRING;
-
- try {
- String indexKey = type.getQualifiedAttributeName(attrName);
- AtlasType attributeType = type.getAttributeType(attrName);
-
- switch (context.getIndexResultType()) {
- case TAG:
- context.addTagSearchAttribute(indexKey);
- break;
-
- case ENTITY:
- context.addEntitySearchAttribute(indexKey);
- break;
-
- default:
- // Do nothing
- }
-
- if (attributeType != null && AtlasTypeUtil.isBuiltInType(attributeType.getTypeName()) && context.getIndexedKeys().contains(indexKey)) {
- if (operatorMap.get(op) != null) {
- // If there's a chance of multi-value then we need some additional processing here
- switch (context.getIndexResultType()) {
- case TAG:
- context.addProcessedTagAttribute(indexKey);
- break;
-
- case ENTITY:
- context.addProcessedEntityAttribute(indexKey);
- break;
- }
-
- ret = String.format(operatorMap.get(op), indexKey, attrVal);
- }
- }
- } catch (AtlasBaseException ex) {
- LOG.warn(ex.getMessage());
- }
-
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 4ac00a3..f4257be 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -128,6 +128,10 @@ public final class EntityGraphRetriever {
return ret;
}
+ public AtlasEntityHeader toAtlasEntityHeader(String guid) throws AtlasBaseException {
+ return toAtlasEntityHeader(getEntityVertex(guid));
+ }
+
public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex entityVertex) throws AtlasBaseException {
return toAtlasEntityHeader(entityVertex, Collections.<String>emptySet());
}
@@ -233,14 +237,15 @@ public final class EntityGraphRetriever {
if (CollectionUtils.isNotEmpty(attributes)) {
for (String attrName : attributes) {
+ String nonQualifiedAttrName = toNonQualifiedName(attrName);
if (ret.hasAttribute(attrName)) {
continue;
}
- Object attrValue = getVertexAttribute(entityVertex, entityType.getAttribute(attrName));
+ Object attrValue = getVertexAttribute(entityVertex, entityType.getAttribute(nonQualifiedAttrName));
if (attrValue != null) {
- ret.setAttribute(attrName, attrValue);
+ ret.setAttribute(nonQualifiedAttrName, attrValue);
}
}
}
@@ -249,6 +254,17 @@ public final class EntityGraphRetriever {
return ret;
}
+ private String toNonQualifiedName(String attrName) {
+ String ret;
+ if (attrName.contains(".")) {
+ String[] attributeParts = attrName.split("\\.");
+ ret = attributeParts[attributeParts.length - 1];
+ } else {
+ ret = attrName;
+ }
+ return ret;
+ }
+
private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) {
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping system attributes for type {}", entity.getTypeName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/util/SearchTracker.java b/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
index 15a8c20..e58e855 100644
--- a/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
+++ b/repository/src/main/java/org/apache/atlas/util/SearchTracker.java
@@ -18,7 +18,7 @@
package org.apache.atlas.util;
import org.apache.atlas.annotation.AtlasService;
-import org.apache.atlas.discovery.SearchPipeline.PipelineContext;
+import org.apache.atlas.discovery.SearchContext;
import java.util.HashMap;
import java.util.Map;
@@ -26,13 +26,13 @@ import java.util.Set;
@AtlasService
public class SearchTracker {
- private Map<String, PipelineContext> activeSearches = new HashMap<>();
+ private Map<String, SearchContext> activeSearches = new HashMap<>();
/**
*
* @param context
*/
- public String add(PipelineContext context) {
+ public String add(SearchContext context) {
String searchId = Thread.currentThread().getName();
activeSearches.put(searchId, context);
@@ -45,13 +45,13 @@ public class SearchTracker {
* @param searchId
* @return
*/
- public PipelineContext terminate(String searchId) {
- PipelineContext ret = null;
+ public SearchContext terminate(String searchId) {
+ SearchContext ret = null;
if (activeSearches.containsKey(searchId)) {
- PipelineContext pipelineToTerminate = activeSearches.remove(searchId);
+ SearchContext pipelineToTerminate = activeSearches.remove(searchId);
- pipelineToTerminate.setForceTerminate(true);
+ pipelineToTerminate.terminateSearch(true);
ret = pipelineToTerminate;
}
@@ -59,7 +59,7 @@ public class SearchTracker {
return ret;
}
- public PipelineContext remove(String id) {
+ public SearchContext remove(String id) {
return activeSearches.remove(id);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/repository/src/test/java/org/apache/atlas/TestModules.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java
index d0da030..4bc1487 100644
--- a/repository/src/test/java/org/apache/atlas/TestModules.java
+++ b/repository/src/test/java/org/apache/atlas/TestModules.java
@@ -143,10 +143,7 @@ public class TestModules {
typeDefChangeListenerMultibinder.addBinding().to(DefaultMetadataService.class);
typeDefChangeListenerMultibinder.addBinding().to(GraphBackedSearchIndexer.class).asEagerSingleton();
- bind(SearchPipeline.class).asEagerSingleton();
bind(SearchTracker.class).asEagerSingleton();
- bind(SolrStep.class).asEagerSingleton();
- bind(GremlinStep.class).asEagerSingleton();
bind(AtlasEntityStore.class).to(AtlasEntityStoreV1.class);
bind(AtlasRelationshipStore.class).to(AtlasRelationshipStoreV1.class);
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 1a9f57a..2a5fc90 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -25,7 +25,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.authorize.AtlasActionTypes;
import org.apache.atlas.authorize.AtlasResourceTypes;
import org.apache.atlas.authorize.simple.AtlasAuthorizationUtils;
-import org.apache.atlas.discovery.SearchPipeline;
+import org.apache.atlas.discovery.SearchContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
@@ -434,7 +434,7 @@ public class AdminResource {
@Path("activeSearches/{id}")
@Produces(Servlets.JSON_MEDIA_TYPE)
public boolean terminateActiveSearch(@PathParam("id") String searchId) {
- SearchPipeline.PipelineContext terminate = activeSearches.terminate(searchId);
+ SearchContext terminate = activeSearches.terminate(searchId);
return null != terminate;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bcec42e3/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index dde300e..efab72a 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -249,7 +249,7 @@ public class DiscoveryREST {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified without tag name");
}
- return atlasDiscoveryService.searchUsingBasicQuery(parameters);
+ return atlasDiscoveryService.searchWithParameters(parameters);
} finally {
AtlasPerfTracer.log(perf);
}
[27/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering (# 3)
Posted by ma...@apache.org.
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 3)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f74e43c2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f74e43c2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f74e43c2
Branch: refs/heads/feature-odf
Commit: f74e43c2bc3fde30cb1d707f1e7c08b8770ec67e
Parents: 5527afb
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Wed Jul 19 11:06:42 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 19 17:42:30 2017 -0700
----------------------------------------------------------------------
.../ClassificationSearchProcessor.java | 45 +++++-----
.../atlas/discovery/EntitySearchProcessor.java | 62 +++++++------
.../discovery/FullTextSearchProcessor.java | 38 +++++---
.../apache/atlas/discovery/SearchProcessor.java | 91 ++++++++++++--------
.../store/graph/v1/AtlasGraphUtilsV1.java | 4 +
5 files changed, 145 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/f74e43c2/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 77b2c7c..b6e0de5 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -58,7 +58,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (useSolrSearch) {
StringBuilder solrQuery = new StringBuilder();
- constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ constructTypeTestQuery(solrQuery, classificationType, typeAndSubTypes);
constructFilterQuery(solrQuery, classificationType, filterCriteria, solrAttributes);
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
@@ -95,20 +95,26 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
try {
- int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
- int limit = context.getSearchParameters().getLimit();
- int resultIdx = qryOffset;
- Set<String> processedGuids = new HashSet<>();
+ final int startIdx = context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ int qryOffset = nextProcessor == null ? startIdx : 0;
+ int resultIdx = qryOffset;
+
+ final Set<String> processedGuids = new HashSet<>();
+ final List<AtlasVertex> entityVertices = new ArrayList<>();
+ final List<AtlasVertex> classificationVertices = new ArrayList<>();
+
+
+ for (; ret.size() < limit; qryOffset += limit) {
+ entityVertices.clear();
+ classificationVertices.clear();
- while (ret.size() < limit) {
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
- List<AtlasVertex> classificationVertices;
-
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
@@ -116,7 +122,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
break;
}
- classificationVertices = getVerticesFromIndexQueryResult(queryResult);
+ getVerticesFromIndexQueryResult(queryResult, classificationVertices);
} else {
Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
@@ -124,13 +130,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
break;
}
- classificationVertices = getVertices(queryResult);
+ getVertices(queryResult, classificationVertices);
}
- qryOffset += limit;
-
- List<AtlasVertex> entityVertices = new ArrayList<>();
-
for (AtlasVertex classificationVertex : classificationVertices) {
Iterable<AtlasEdge> edges = classificationVertex.getEdges(AtlasEdgeDirection.IN);
@@ -148,12 +150,12 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
}
- entityVertices = super.filter(entityVertices);
+ super.filter(entityVertices);
for (AtlasVertex entityVertex : entityVertices) {
resultIdx++;
- if (resultIdx < context.getSearchParameters().getOffset()) {
+ if (resultIdx <= startIdx) {
continue;
}
@@ -176,7 +178,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
@Override
- public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
+ public void filter(List<AtlasVertex> entityVertices) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
@@ -185,14 +187,13 @@ public class ClassificationSearchProcessor extends SearchProcessor {
query.addConditionsFrom(filterGraphQuery);
- List<AtlasVertex> ret = getVertices(query.vertices().iterator());
+ entityVertices.clear();
+ getVertices(query.vertices().iterator(), entityVertices);
- ret = super.filter(ret);
+ super.filter(entityVertices);
if (LOG.isDebugEnabled()) {
- LOG.debug("<== ClassificationSearchProcessor.filter({}): ret.size()={}", entityVertices.size(), ret.size());
+ LOG.debug("<== ClassificationSearchProcessor.filter(): ret.size()={}", entityVertices.size());
}
-
- return ret;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f74e43c2/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 50376ef..6f629eb 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
@@ -59,7 +60,7 @@ public class EntitySearchProcessor extends SearchProcessor {
StringBuilder solrQuery = new StringBuilder();
if (typeSearchBySolr) {
- constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ constructTypeTestQuery(solrQuery, entityType, typeAndSubTypes);
}
if (attrSearchBySolr) {
@@ -127,34 +128,48 @@ public class EntitySearchProcessor extends SearchProcessor {
}
try {
- int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery == null)) ? context.getSearchParameters().getOffset() : 0;
- int limit = context.getSearchParameters().getLimit();
- int resultIdx = qryOffset;
+ final int startIdx = context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery == null)) ? startIdx : 0;
+ int resultIdx = qryOffset;
+
+ final List<AtlasVertex> entityVertices = new ArrayList<>();
+
+ for (; ret.size() < limit; qryOffset += limit) {
+ entityVertices.clear();
- while (ret.size() < limit) {
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
break;
}
- List<AtlasVertex> vertices;
-
if (indexQuery != null) {
- Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
+ Iterator<AtlasIndexQuery.Result> idxQueryResult = indexQuery.vertices(qryOffset, limit);
- if (!queryResult.hasNext()) { // no more results from solr - end of search
+ if (!idxQueryResult.hasNext()) { // no more results from solr - end of search
break;
}
- vertices = getVerticesFromIndexQueryResult(queryResult);
+ while (idxQueryResult.hasNext()) {
+ AtlasVertex vertex = idxQueryResult.next().getVertex();
+
+ // skip non-entity vertices
+ if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) {
+ LOG.warn("EntitySearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId()); // might cause duplicate entries in result
+
+ continue;
+ }
+
+ entityVertices.add(vertex);
+ }
if (graphQuery != null) {
- AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices));
+ AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
guidQuery.addConditionsFrom(graphQuery);
- vertices = getVertices(guidQuery.vertices().iterator());
+ getVertices(guidQuery.vertices().iterator(), entityVertices);
}
} else {
Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset, limit).iterator();
@@ -163,21 +178,19 @@ public class EntitySearchProcessor extends SearchProcessor {
break;
}
- vertices = getVertices(queryResult);
+ getVertices(queryResult, entityVertices);
}
- qryOffset += limit;
-
- vertices = super.filter(vertices);
+ super.filter(entityVertices);
- for (AtlasVertex vertex : vertices) {
+ for (AtlasVertex entityVertex : entityVertices) {
resultIdx++;
- if (resultIdx < context.getSearchParameters().getOffset()) {
+ if (resultIdx <= startIdx) {
continue;
}
- ret.add(vertex);
+ ret.add(entityVertex);
if (ret.size() == limit) {
break;
@@ -196,7 +209,7 @@ public class EntitySearchProcessor extends SearchProcessor {
}
@Override
- public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
+ public void filter(List<AtlasVertex> entityVertices) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> EntitySearchProcessor.filter({})", entityVertices.size());
}
@@ -205,14 +218,13 @@ public class EntitySearchProcessor extends SearchProcessor {
query.addConditionsFrom(filterGraphQuery);
- List<AtlasVertex> ret = getVertices(query.vertices().iterator());
+ entityVertices.clear();
+ getVertices(query.vertices().iterator(), entityVertices);
- ret = super.filter(ret);
+ super.filter(entityVertices);
if (LOG.isDebugEnabled()) {
- LOG.debug("<== EntitySearchProcessor.filter({}): ret.size()={}", entityVertices.size(), ret.size());
+ LOG.debug("<== EntitySearchProcessor.filter(): ret.size()={}", entityVertices.size());
}
-
- return ret;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f74e43c2/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 83368c2..22d91e0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -19,8 +19,10 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
@@ -68,7 +70,7 @@ public class FullTextSearchProcessor extends SearchProcessor {
queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
} else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
- context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+ context.getClassificationType().getTypeName(), typeAndSubTypeNames.size());
}
}
@@ -92,11 +94,16 @@ public class FullTextSearchProcessor extends SearchProcessor {
}
try {
- int qryOffset = nextProcessor == null ? context.getSearchParameters().getOffset() : 0;
- int limit = context.getSearchParameters().getLimit();
- int resultIdx = qryOffset;
+ final int startIdx = context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ int qryOffset = nextProcessor == null ? startIdx : 0;
+ int resultIdx = qryOffset;
+
+ final List<AtlasVertex> entityVertices = new ArrayList<>();
+
+ for (; ret.size() < limit; qryOffset += limit) {
+ entityVertices.clear();
- while (ret.size() < limit) {
if (context.terminateSearch()) {
LOG.warn("query terminated: {}", context.getSearchParameters());
@@ -109,20 +116,29 @@ public class FullTextSearchProcessor extends SearchProcessor {
break;
}
- qryOffset += limit;
+ while (idxQueryResult.hasNext()) {
+ AtlasVertex vertex = idxQueryResult.next().getVertex();
- List<AtlasVertex> vertices = getVerticesFromIndexQueryResult(idxQueryResult);
+ // skip non-entity vertices
+ if (!AtlasGraphUtilsV1.isEntityVertex(vertex)) {
+ LOG.warn("FullTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId()); // might cause duplicate entries in result
+
+ continue;
+ }
+
+ entityVertices.add(vertex);
+ }
- vertices = super.filter(vertices);
+ super.filter(entityVertices);
- for (AtlasVertex vertex : vertices) {
+ for (AtlasVertex entityVertex : entityVertices) {
resultIdx++;
- if (resultIdx < context.getSearchParameters().getOffset()) {
+ if (resultIdx <= startIdx) {
continue;
}
- ret.add(vertex);
+ ret.add(entityVertex);
if (ret.size() == limit) {
break;
http://git-wip-us.apache.org/repos/asf/atlas/blob/f74e43c2/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 596b43b..2e75dfe 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -50,6 +50,7 @@ public abstract class SearchProcessor {
public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "( ";
public static final String BRACE_CLOSE_STR = " )";
+ public static final char DOUBLE_QUOTE = '"';
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final char[] OFFENDING_CHARS = {'@', '/', ' '}; // This can grow as we discover corner cases
@@ -87,8 +88,10 @@ public abstract class SearchProcessor {
public abstract List<AtlasVertex> execute();
- public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
- return nextProcessor == null || CollectionUtils.isEmpty(entityVertices) ? entityVertices : nextProcessor.filter(entityVertices);
+ public void filter(List<AtlasVertex> entityVertices) {
+ if (nextProcessor != null && CollectionUtils.isNotEmpty(entityVertices)) {
+ nextProcessor.filter(entityVertices);
+ }
}
@@ -178,12 +181,26 @@ public abstract class SearchProcessor {
return ret;
}
- protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
+ protected void constructTypeTestQuery(StringBuilder solrQuery, AtlasStructType type, Set<String> typeAndAllSubTypes) {
String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
- solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": (")
- .append(typeAndSubtypesString)
- .append(")");
+ if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": (")
+ .append(typeAndSubtypesString)
+ .append(")");
+ }
+
+ if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
+ }
}
protected void constructFilterQuery(StringBuilder solrQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> solrAttributes) {
@@ -200,14 +217,6 @@ public abstract class SearchProcessor {
solrQuery.append(filterQuery);
}
}
-
- if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
- if (solrQuery.length() > 0) {
- solrQuery.append(AND_STR);
- }
-
- solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
- }
}
private String toSolrQuery(AtlasStructType type, FilterCriteria criteria, Set<String> solrAttributes, int level) {
@@ -246,15 +255,10 @@ public abstract class SearchProcessor {
String ret = EMPTY_STRING;
try {
- String qualifiedName = type.getQualifiedAttributeName(attrName);
-
if (OPERATOR_MAP.get(op) != null) {
- if (hasOffendingChars(attrVal)) {
- // FIXME: if attrVal has offending chars & op is contains, endsWith, startsWith, solr doesn't like it and results are skewed
- ret = String.format(OPERATOR_MAP.get(op), qualifiedName, "\"" + attrVal + "\"");
- } else {
- ret = String.format(OPERATOR_MAP.get(op), qualifiedName, attrVal);
- }
+ String qualifiedName = type.getQualifiedAttributeName(attrName);
+
+ ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue(attrVal));
}
} catch (AtlasBaseException ex) {
LOG.warn(ex.getMessage());
@@ -348,32 +352,28 @@ public abstract class SearchProcessor {
private String getLikeRegex(String attributeValue) { return ".*" + attributeValue + ".*"; }
- protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult) {
- List<AtlasVertex> ret = new ArrayList<>();
-
+ protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, List<AtlasVertex> vertices) {
if (idxQueryResult != null) {
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
- ret.add(vertex);
+ vertices.add(vertex);
}
}
- return ret;
+ return vertices;
}
- protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> vertices) {
- List<AtlasVertex> ret = new ArrayList<>();
-
- if (vertices != null) {
- while (vertices.hasNext()) {
- AtlasVertex vertex = vertices.next();
+ protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, List<AtlasVertex> vertices) {
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ AtlasVertex vertex = iterator.next();
- ret.add(vertex);
+ vertices.add(vertex);
}
}
- return ret;
+ return vertices;
}
protected Set<String> getGuids(List<AtlasVertex> vertices) {
@@ -402,7 +402,24 @@ public abstract class SearchProcessor {
return defaultValue;
}
- private boolean hasOffendingChars(String str) {
- return StringUtils.containsAny(str, OFFENDING_CHARS);
+ private String escapeIndexQueryValue(String value) {
+ String ret = value;
+
+ if (StringUtils.containsAny(value, OFFENDING_CHARS)) {
+ boolean isQuoteAtStart = value.charAt(0) == DOUBLE_QUOTE;
+ boolean isQuoteAtEnd = value.charAt(value.length() - 1) == DOUBLE_QUOTE;
+
+ if (!isQuoteAtStart) {
+ if (!isQuoteAtEnd) {
+ ret = DOUBLE_QUOTE + value + DOUBLE_QUOTE;
+ } else {
+ ret = DOUBLE_QUOTE + value;
+ }
+ } else if (!isQuoteAtEnd) {
+ ret = value + DOUBLE_QUOTE;
+ }
+ }
+
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f74e43c2/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
index cd9a47a..43f2c55 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
@@ -103,6 +103,10 @@ public class AtlasGraphUtilsV1 {
}
}
+ public static boolean isEntityVertex(AtlasVertex vertex) {
+ return StringUtils.isNotEmpty(getIdFromVertex(vertex)) && StringUtils.isNotEmpty(getTypeName(vertex));
+ }
+
public static boolean isReference(AtlasType type) {
return isReference(type.getTypeCategory());
}
[20/39] atlas git commit: ATLAS-1956: AtlasSearchResult should
include attributes present in filter criteria
Posted by ma...@apache.org.
ATLAS-1956: AtlasSearchResult should include attributes present in filter criteria
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/d7ce0a53
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/d7ce0a53
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/d7ce0a53
Branch: refs/heads/feature-odf
Commit: d7ce0a535621e2cd1176ff3e069ce60b27b04c17
Parents: abc4856
Author: apoorvnaik <ap...@apache.org>
Authored: Mon Jul 17 18:07:15 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jul 17 18:07:15 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/discovery/EntityDiscoveryService.java | 4 ++++
.../src/main/java/org/apache/atlas/discovery/SearchContext.java | 5 +++++
.../main/java/org/apache/atlas/discovery/SearchProcessor.java | 5 +++++
3 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/d7ce0a53/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index c7a624f..dbe11a6 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -420,6 +420,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
resultAttributes.addAll(searchParameters.getAttributes());
}
+ if (CollectionUtils.isNotEmpty(context.getEntityAttributes())) {
+ resultAttributes.addAll(context.getEntityAttributes());
+ }
+
for (String resultAttribute : resultAttributes) {
AtlasAttribute attribute = context.getEntityType().getAttribute(resultAttribute);
http://git-wip-us.apache.org/repos/asf/atlas/blob/d7ce0a53/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 2125d61..55a07f3 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -27,6 +27,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import java.util.HashSet;
import java.util.Set;
@@ -35,6 +36,7 @@ public class SearchContext {
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
private final Set<String> indexedKeys;
+ private final Set<String> entityAttributes;
private final AtlasEntityType entityType;
private final AtlasClassificationType classificationType;
private SearchProcessor searchProcessor;
@@ -45,6 +47,7 @@ public class SearchContext {
this.typeRegistry = typeRegistry;
this.graph = graph;
this.indexedKeys = indexedKeys;
+ this.entityAttributes = new HashSet<>();
this.entityType = typeRegistry.getEntityTypeByName(searchParameters.getTypeName());
this.classificationType = typeRegistry.getClassificationTypeByName(searchParameters.getClassification());
@@ -70,6 +73,8 @@ public class SearchContext {
public Set<String> getIndexedKeys() { return indexedKeys; }
+ public Set<String> getEntityAttributes() { return entityAttributes; }
+
public AtlasEntityType getEntityType() { return entityType; }
public AtlasClassificationType getClassificationType() { return classificationType; }
http://git-wip-us.apache.org/repos/asf/atlas/blob/d7ce0a53/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 29430ef..1481809 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -118,6 +118,11 @@ public abstract class SearchProcessor {
gremlinFiltered.add(attributeName);
}
+ if (structType instanceof AtlasEntityType) {
+ // Capture the entity attributes
+ context.getEntityAttributes().add(attributeName);
+ }
+
allAttributes.add(attributeName);
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
[07/39] atlas git commit: ATLAS-1896:Fix for adding Cross Origin
[CORS] headers for Atlas
Posted by ma...@apache.org.
ATLAS-1896:Fix for adding Cross Origin [CORS] headers for Atlas
Signed-off-by: nixonrodrigues <ni...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/40d909ef
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/40d909ef
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/40d909ef
Branch: refs/heads/feature-odf
Commit: 40d909ef8726b807a263ff8f8be57150e51f910d
Parents: 4e62a06
Author: rdsolani <rd...@gmail.com>
Authored: Wed Jul 12 12:35:07 2017 +0530
Committer: nixonrodrigues <ni...@apache.org>
Committed: Wed Jul 12 16:49:40 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/atlas/utils/MD5Utils.java | 66 -------------------
.../org/apache/atlas/utils/SHA256Utils.java | 68 ++++++++++++++++++++
distro/src/conf/atlas-application.properties | 7 ++
.../graph/TypedInstanceToGraphMapper.java | 4 +-
.../apache/atlas/typesystem/persistence/Id.java | 4 +-
.../persistence/ReferenceableInstance.java | 4 +-
.../typesystem/persistence/StructInstance.java | 4 +-
.../web/filters/AtlasAuthenticationFilter.java | 25 +++++--
8 files changed, 103 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/common/src/main/java/org/apache/atlas/utils/MD5Utils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/utils/MD5Utils.java b/common/src/main/java/org/apache/atlas/utils/MD5Utils.java
deleted file mode 100644
index 8529b3c..0000000
--- a/common/src/main/java/org/apache/atlas/utils/MD5Utils.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.utils;
-
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-/**
- * Utilities for MD5 hash.
- */
-public final class MD5Utils {
-
- private MD5Utils() {
- }
-
- private static final ThreadLocal<MessageDigest> DIGESTER_FACTORY =
- new ThreadLocal<MessageDigest>() {
- @Override
- protected MessageDigest initialValue() {
- try {
- return MessageDigest.getInstance("MD5");
- } catch (NoSuchAlgorithmException e) {
- throw new RuntimeException(e);
- }
- }
- };
-
- /**
- * Create a thread local MD5 digester.
- */
- public static MessageDigest getDigester() {
- MessageDigest digester = DIGESTER_FACTORY.get();
- digester.reset();
- return digester;
- }
-
- private static final char[] HEX_DIGITS =
- {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};
-
- public static String toString(byte[] digest) {
- StringBuilder buf = new StringBuilder(MD5_LEN * 2);
- for (int i = 0; i < MD5_LEN; i++) {
- int b = digest[i];
- buf.append(HEX_DIGITS[(b >> 4) & 0xf]);
- buf.append(HEX_DIGITS[b & 0xf]);
- }
- return buf.toString();
- }
-
- public static final int MD5_LEN = 16;
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java b/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java
new file mode 100644
index 0000000..0b57068
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/utils/SHA256Utils.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.utils;
+
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class SHA256Utils{
+
+
+ private static final ThreadLocal<MessageDigest> DIGESTER_FACTORY =
+ new ThreadLocal<MessageDigest>() {
+ @Override
+ protected MessageDigest initialValue() {
+ try {
+ return MessageDigest.getInstance("SHA-256");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ /**
+ * Create a thread local SHA256 digester.
+ */
+
+ public static MessageDigest getDigester() {
+ MessageDigest digester = DIGESTER_FACTORY.get();
+ digester.reset();
+ return digester;
+ }
+
+ public static String toString(byte[] hash) {
+ try{
+ StringBuffer hexString = new StringBuffer();
+
+ for (int i = 0; i < hash.length; i++) {
+ String hex = Integer.toHexString(0xff & hash[i]);
+ if(hex.length() == 1) hexString.append('0');
+ hexString.append(hex);
+ }
+
+ return hexString.toString();
+ } catch(Exception ex){
+ throw new RuntimeException(ex);
+ }
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/distro/src/conf/atlas-application.properties
----------------------------------------------------------------------
diff --git a/distro/src/conf/atlas-application.properties b/distro/src/conf/atlas-application.properties
index 474f253..c3213df 100755
--- a/distro/src/conf/atlas-application.properties
+++ b/distro/src/conf/atlas-application.properties
@@ -251,3 +251,10 @@ atlas.metric.query.cache.ttlInSecs=900
#Set to false to disable full text search.
#atlas.search.fulltext.enable=true
+
+
+########## Add http headers ###########
+
+#atlas.headers.Access-Control-Allow-Origin=*
+#atlas.headers.Access-Control-Allow-Methods=GET,OPTIONS,HEAD,PUT,POST
+#atlas.headers.<headerName>=<headerValue>
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
index 4dac293..1fb4ee9 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
@@ -39,7 +39,7 @@ import org.apache.atlas.typesystem.types.*;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
-import org.apache.atlas.utils.MD5Utils;
+import org.apache.atlas.utils.SHA256Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -675,7 +675,7 @@ public final class TypedInstanceToGraphMapper {
}
// Update attributes
- final MessageDigest digester = MD5Utils.getDigester();
+ final MessageDigest digester = SHA256Utils.getDigester();
String newSignature = newAttributeValue.getSignatureHash(digester);
String curSignature = GraphHelper.getSingleValuedProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY, String.class);
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
index 101a1f3..ba05a45 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
@@ -24,7 +24,7 @@ import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.FieldMapping;
-import org.apache.atlas.utils.MD5Utils;
+import org.apache.atlas.utils.SHA256Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -290,7 +290,7 @@ public class Id implements ITypedReferenceableInstance {
digester.update(id.getBytes(Charset.forName("UTF-8")));
digester.update(typeName.getBytes(Charset.forName("UTF-8")));
byte[] digest = digester.digest();
- return MD5Utils.toString(digest);
+ return SHA256Utils.toString(digest);
}
private static long nextNegativeLong() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
index 11cbb26..be2634d 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
@@ -29,7 +29,7 @@ import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.utils.MD5Utils;
+import org.apache.atlas.utils.SHA256Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -130,6 +130,6 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
ClassType classType = TypeSystem.getInstance().getDataType(ClassType.class, getTypeName());
classType.updateSignatureHash(digester, this);
byte[] digest = digester.digest();
- return MD5Utils.toString(digest);
+ return SHA256Utils.toString(digest);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
index fc10d07..766d2d0 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
@@ -32,7 +32,7 @@ import org.apache.atlas.typesystem.types.FieldMapping;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
-import org.apache.atlas.utils.MD5Utils;
+import org.apache.atlas.utils.SHA256Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -774,7 +774,7 @@ public class StructInstance implements ITypedStruct {
StructType structType = TypeSystem.getInstance().getDataType(StructType.class, getTypeName());
structType.updateSignatureHash(digester, this);
byte[] digest = digester.digest();
- return MD5Utils.toString(digest);
+ return SHA256Utils.toString(digest);
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/40d909ef/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java
index 25aede3..444b094 100644
--- a/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java
+++ b/webapp/src/main/java/org/apache/atlas/web/filters/AtlasAuthenticationFilter.java
@@ -26,6 +26,7 @@ import org.apache.atlas.web.security.AtlasAuthenticationProvider;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.collections.iterators.IteratorEnumeration;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationConverter;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -74,6 +75,7 @@ import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+
/**
* This enforces authentication as part of the filter before processing the request.
* todo: Subclass of {@link org.apache.hadoop.security.authentication.server.AuthenticationFilter}.
@@ -90,7 +92,8 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter {
private boolean isInitializedByTomcat;
private Set<Pattern> browserUserAgents;
private boolean supportKeyTabBrowserLogin = false;
-
+ private Configuration configuration;
+ private Properties headerProperties;
public AtlasAuthenticationFilter() {
try {
LOG.info("AtlasAuthenticationFilter initialization started");
@@ -112,8 +115,16 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter {
public void init(FilterConfig filterConfig) throws ServletException {
LOG.info("AtlasAuthenticationFilter initialization started");
final FilterConfig globalConf = filterConfig;
-
final Map<String, String> params = new HashMap<>();
+ try {
+ configuration = ApplicationProperties.get();
+ } catch (Exception e) {
+ throw new ServletException(e);
+ }
+
+ if (configuration != null) {
+ headerProperties = ConfigurationConverter.getProperties(configuration.subset("atlas.headers"));
+ }
FilterConfig filterConfig1 = new FilterConfig() {
@Override
@@ -147,7 +158,6 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter {
optionsServlet = new HttpServlet() {
};
optionsServlet.init();
-
}
@@ -176,7 +186,6 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter {
@Override
protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException {
- Configuration configuration;
try {
configuration = ApplicationProperties.get();
} catch (Exception e) {
@@ -308,10 +317,16 @@ public class AtlasAuthenticationFilter extends AuthenticationFilter {
try {
Authentication existingAuth = SecurityContextHolder.getContext().getAuthentication();
HttpServletResponse httpResponse = (HttpServletResponse) response;
-
AtlasResponseRequestWrapper responseWrapper = new AtlasResponseRequestWrapper(httpResponse);
responseWrapper.setHeader("X-Frame-Options", "DENY");
+ if (headerProperties != null) {
+ for (String headerKey : headerProperties.stringPropertyNames()) {
+ String headerValue = headerProperties.getProperty(headerKey);
+ responseWrapper.setHeader(headerKey, headerValue);
+ }
+ }
+
if (existingAuth == null) {
String authHeader = httpRequest.getHeader("Authorization");
if (authHeader != null && authHeader.startsWith("Basic")) {
[11/39] atlas git commit: ATLAS-1940: fix to remove duplicate type
which causes Atlas server to fail during startup
Posted by ma...@apache.org.
ATLAS-1940: fix to remove duplicate type which causes Atlas server to fail during startup
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/de916084
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/de916084
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/de916084
Branch: refs/heads/feature-odf
Commit: de91608446514df864995d1e50a8db4276b08ad8
Parents: c3808cf
Author: nixonrodrigues <ni...@apache.org>
Authored: Fri Jul 14 19:16:23 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 14 14:53:37 2017 -0700
----------------------------------------------------------------------
.../store/graph/AtlasTypeDefGraphStore.java | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/de916084/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index 9d9c59d..eedcd10 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -159,6 +159,7 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
return ret;
}
+
@Override
public AtlasRelationshipDef getRelationshipDefByName(String name) throws AtlasBaseException {
AtlasRelationshipDef ret = typeRegistry.getRelationshipDefByName(name);
@@ -680,21 +681,37 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
for (AtlasStructDef structDef : typesDef.getStructDefs()) {
rectifyAttributesIfNeeded(entityNames, structDef);
}
+ removeDuplicateTypeIfAny(typesDef.getStructDefs());
}
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
rectifyAttributesIfNeeded(entityNames, classificationDef);
}
+ removeDuplicateTypeIfAny(typesDef.getClassificationDefs());
}
if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
rectifyAttributesIfNeeded(entityNames, entityDef);
}
+ removeDuplicateTypeIfAny(typesDef.getEntityDefs());
}
}
+ private <T extends AtlasBaseTypeDef> void removeDuplicateTypeIfAny(List<T> defList) {
+ final Set<String> entityDefNames = new HashSet<>();
+
+ for (int i = 0; i < defList.size(); i++) {
+ if (!entityDefNames.add((defList.get(i)).getName())) {
+ LOG.warn(" Found Duplicate Type => " + defList.get(i).getName());
+ defList.remove(i);
+ i--;
+ }
+ }
+ }
+
+
private void rectifyAttributesIfNeeded(final Set<String> entityNames, AtlasStructDef structDef) {
List<AtlasAttributeDef> attributeDefs = structDef.getAttributeDefs();
[15/39] atlas git commit: ATLAS-1952: UI optimization to use
searchResult.referredEntity to render object type attributes
Posted by ma...@apache.org.
ATLAS-1952: UI optimization to use searchResult.referredEntity to render object type attributes
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/84c6d52d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/84c6d52d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/84c6d52d
Branch: refs/heads/feature-odf
Commit: 84c6d52d266c6aa2039e7e8cb9a7edb8f4bb8a93
Parents: bcec42e
Author: kevalbhatt <kb...@apache.org>
Authored: Fri Jul 14 17:47:03 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 14 16:08:31 2017 -0700
----------------------------------------------------------------------
dashboardv2/public/js/collection/VSearchList.js | 3 ++-
dashboardv2/public/js/utils/CommonViewFunction.js | 17 ++---------------
dashboardv2/public/js/utils/Utils.js | 17 ++++++++++++++++-
.../js/views/entity/EntityDetailTableLayoutView.js | 7 ++++---
.../js/views/search/SearchResultLayoutView.js | 5 ++++-
5 files changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6d52d/dashboardv2/public/js/collection/VSearchList.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/collection/VSearchList.js b/dashboardv2/public/js/collection/VSearchList.js
index 6291838..0a7b713 100644
--- a/dashboardv2/public/js/collection/VSearchList.js
+++ b/dashboardv2/public/js/collection/VSearchList.js
@@ -37,6 +37,7 @@ define(['require',
parseRecords: function(resp, options) {
this.queryType = resp.queryType;
this.queryText = resp.queryText;
+ this.referredEntities = resp.referredEntities;
return resp.entities ? resp.entities : [];
},
getBasicRearchResult: function(options) {
@@ -62,4 +63,4 @@ define(['require',
}
);
return VSearchList;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6d52d/dashboardv2/public/js/utils/CommonViewFunction.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/utils/CommonViewFunction.js b/dashboardv2/public/js/utils/CommonViewFunction.js
index a04fb3f..6b8f50d 100644
--- a/dashboardv2/public/js/utils/CommonViewFunction.js
+++ b/dashboardv2/public/js/utils/CommonViewFunction.js
@@ -75,19 +75,6 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
}
});
};
- CommonViewFunction.findAndmergeRefEntity = function(attributeObject, referredEntities) {
- _.each(attributeObject, function(obj, key) {
- if (_.isObject(obj)) {
- if (_.isArray(obj)) {
- _.each(obj, function(value) {
- _.extend(value, referredEntities[value.guid]);
- });
- } else {
- _.extend(obj, referredEntities[obj.guid]);
- }
- }
- });
- }
CommonViewFunction.propertyTable = function(options) {
var scope = options.scope,
valueObject = options.valueObject,
@@ -179,7 +166,7 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
}
if (id && inputOutputField) {
var name = Utils.getName(inputOutputField);
- if (name === "-" || name === id) {
+ if ((name === "-" || name === id) && !inputOutputField.attributes) {
var fetch = true;
var fetchId = (_.isObject(id) ? id.id : id);
fetchInputOutputValue(fetchId);
@@ -474,4 +461,4 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
}
}
return CommonViewFunction;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6d52d/dashboardv2/public/js/utils/Utils.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/utils/Utils.js b/dashboardv2/public/js/utils/Utils.js
index 9071df3..6e0453b 100644
--- a/dashboardv2/public/js/utils/Utils.js
+++ b/dashboardv2/public/js/utils/Utils.js
@@ -454,6 +454,21 @@ define(['require', 'utils/Globals', 'pnotify', 'utils/Messages', 'pnotify.button
loaderEl.hide();
titleBoxEl.fadeIn();
}
+ Utils.findAndMergeRefEntity = function(attributeObject, referredEntities) {
+ if (attributeObject && referredEntities) {
+ _.each(attributeObject, function(obj, key) {
+ if (_.isObject(obj)) {
+ if (_.isArray(obj)) {
+ _.each(obj, function(value) {
+ _.extend(value, referredEntities[value.guid]);
+ });
+ } else {
+ _.extend(obj, referredEntities[obj.guid]);
+ }
+ }
+ });
+ }
+ }
Utils.getNestedSuperTypeObj = function(options) {
var flag = 0,
data = options.data,
@@ -539,4 +554,4 @@ define(['require', 'utils/Globals', 'pnotify', 'utils/Messages', 'pnotify.button
});
return Utils;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6d52d/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js b/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
index 87c5dcd..e27edf4 100644
--- a/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
+++ b/dashboardv2/public/js/views/entity/EntityDetailTableLayoutView.js
@@ -21,7 +21,8 @@ define(['require',
'hbs!tmpl/entity/EntityDetailTableLayoutView_tmpl',
'utils/CommonViewFunction',
'models/VEntity',
-], function(require, Backbone, EntityDetailTableLayoutView_tmpl, CommonViewFunction, VEntity) {
+ 'utils/Utils'
+], function(require, Backbone, EntityDetailTableLayoutView_tmpl, CommonViewFunction, VEntity, Utils) {
'use strict';
var EntityDetailTableLayoutView = Backbone.Marionette.LayoutView.extend(
@@ -58,7 +59,7 @@ define(['require',
entityTableGenerate: function() {
var that = this,
attributeObject = this.entity.attributes;
- CommonViewFunction.findAndmergeRefEntity(attributeObject, that.referredEntities);
+ Utils.findAndMergeRefEntity(attributeObject, that.referredEntities);
if (attributeObject && attributeObject.columns) {
var valueSorted = _.sortBy(attributeObject.columns, function(val) {
return val.attributes.position
@@ -70,4 +71,4 @@ define(['require',
}
});
return EntityDetailTableLayoutView;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6d52d/dashboardv2/public/js/views/search/SearchResultLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchResultLayoutView.js b/dashboardv2/public/js/views/search/SearchResultLayoutView.js
index dba27fa..319beab 100644
--- a/dashboardv2/public/js/views/search/SearchResultLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchResultLayoutView.js
@@ -352,6 +352,7 @@ define(['require',
return;
}
if (isPostMethod) {
+ that.searchCollection.referredEntities = model.referredEntities;
that.searchCollection.reset(model.entities);
}
if (that.searchCollection.models.length === 0 && that.offset > that.limit) {
@@ -658,7 +659,9 @@ define(['require',
'valueObject': {},
'isTable': false
}
+
tempObj.valueObject[key] = modelObj.attributes[key]
+ Utils.findAndMergeRefEntity(tempObj.valueObject, that.searchCollection.referredEntities);
return CommonViewFunction.propertyTable(tempObj);
}
}
@@ -862,4 +865,4 @@ define(['require',
}
});
return SearchResultLayoutView;
-});
+});
\ No newline at end of file
[16/39] atlas git commit: ATLAS-1939: added parameter validation to
prevent NPE during import
Posted by ma...@apache.org.
ATLAS-1939: added parameter validation to prevent NPE during import
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/eddab3b1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/eddab3b1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/eddab3b1
Branch: refs/heads/feature-odf
Commit: eddab3b12a1318f6e787d062816e28479715f077
Parents: 84c6d52
Author: Sharmadha Sainath <ss...@hortonworks.com>
Authored: Sat Jul 15 01:16:31 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sat Jul 15 14:22:41 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/atlas/web/resources/AdminResource.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/eddab3b1/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 2a5fc90..fe9111b 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -370,6 +370,10 @@ public class AdminResource {
AtlasImportResult result;
try {
+ if (StringUtils.isEmpty(jsonData)) {
+ jsonData = "{}";
+ }
+
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
ZipSource zipSource = new ZipSource(inputStream);
[05/39] atlas git commit: ATLAS-1945: updated UI to use fixed version
of plugins
Posted by ma...@apache.org.
ATLAS-1945: updated UI to use fixed version of plugins
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/ea1c3b68
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/ea1c3b68
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/ea1c3b68
Branch: refs/heads/feature-odf
Commit: ea1c3b68ee8882f46cec1f9394b2aadad1f80f7a
Parents: ad39839
Author: kevalbhatt <kb...@apache.org>
Authored: Wed Jul 12 11:53:52 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 12 03:10:06 2017 -0700
----------------------------------------------------------------------
dashboardv2/package.json | 54 +++++++++++++++++++++----------------------
1 file changed, 26 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/ea1c3b68/dashboardv2/package.json
----------------------------------------------------------------------
diff --git a/dashboardv2/package.json b/dashboardv2/package.json
index 59889d3..5be682e 100644
--- a/dashboardv2/package.json
+++ b/dashboardv2/package.json
@@ -1,7 +1,6 @@
{
"name": "atlas-metadata",
"description": "Apache Atlas",
- "version": "0.5.0-incubating",
"private": true,
"repository": {
"type": "git",
@@ -17,34 +16,33 @@
],
"dependencies": {
"backbone": "1.2.3",
- "backbone.babysitter": "git+https://github.com/marionettejs/backbone.babysitter.git",
+ "backbone.babysitter": "0.1.12",
"backbone.marionette": "2.3.2",
- "backbone.paginator": "^2.0.0",
- "backbone.wreqr": "^1.3.2",
- "backgrid": "^0.3.7",
- "backgrid-filter": "^0.3.7",
- "backgrid-orderable-columns": "^0.1.2",
- "backgrid-paginator": "^0.3.7",
- "backgrid-select-all": "^0.3.5",
- "backgrid-sizeable-columns": "^0.1.1",
- "bootstrap": "^3.3.5",
- "bootstrap-daterangepicker": "^2.1.25",
- "d3": "^3.5.16",
- "d3-tip": "^0.6.7",
- "dagre-d3": "^0.4.17",
- "font-awesome": "^4.6.3",
- "jQuery-QueryBuilder": "^2.4.3",
- "jquery": "^2.2.2",
- "jquery-asBreadcrumbs": "^0.2.1",
- "jquery-placeholder": "^2.3.1",
- "jquery-ui": "^1.10.4",
- "moment": "^2.15.0",
- "platform": "^1.3.1",
- "pnotify": "^3.0.0",
- "requirejs": "^2.1.16",
- "requirejs-text": "^2.0.12",
- "select2": "^4.0.3",
- "underscore": "^1.8.3"
+ "backbone.paginator": "2.0.5",
+ "backbone.wreqr": "1.4.0",
+ "backgrid": "0.3.8",
+ "backgrid-filter": "0.3.7",
+ "backgrid-orderable-columns": "0.1.2",
+ "backgrid-paginator": "0.3.9",
+ "backgrid-select-all": "0.3.5",
+ "backgrid-sizeable-columns": "0.1.1",
+ "bootstrap": "3.3.7",
+ "bootstrap-daterangepicker": "2.1.25",
+ "d3": "3.5.17",
+ "d3-tip": "0.6.8",
+ "dagre-d3": "0.4.17",
+ "font-awesome": "4.7.0",
+ "jQuery-QueryBuilder": "2.4.3",
+ "jquery": "2.2.4",
+ "jquery-asBreadcrumbs": "0.2.2",
+ "jquery-placeholder": "2.3.1",
+ "moment": "2.18.1",
+ "platform": "1.3.4",
+ "pnotify": "3.2.0",
+ "requirejs": "2.3.3",
+ "requirejs-text": "2.0.15",
+ "select2": "4.0.3",
+ "underscore": "1.8.3"
},
"devDependencies": {
"connect-livereload": "^0.5.4",
[02/39] atlas git commit: ATLAS-1907: created RelationshipDefs for
all base models and use it when creating edges for entities
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index a5b5730..e919d1c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -29,6 +29,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
@@ -49,6 +51,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -65,6 +68,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
+import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
public final class EntityGraphRetriever {
@@ -180,6 +184,8 @@ public final class EntityGraphRetriever {
mapAttributes(entityVertex, entity, entityExtInfo);
+ mapRelationshipAttributes(entityVertex, entity, entityExtInfo);
+
mapClassifications(entityVertex, entity, entityExtInfo);
}
@@ -278,6 +284,23 @@ public final class EntityGraphRetriever {
}
}
+ private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
+ AtlasType objType = typeRegistry.getType(entity.getTypeName());
+
+ if (!(objType instanceof AtlasEntityType)) {
+ throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, entity.getTypeName());
+ }
+
+ AtlasEntityType entityType = (AtlasEntityType) objType;
+
+ for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
+
+ Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attribute, entityExtInfo);
+
+ entity.addRelationshipAttribute(attribute.getName(), attrValue);
+ }
+ }
+
public List<AtlasClassification> getClassifications(String guid) throws AtlasBaseException {
AtlasVertex instanceVertex = AtlasGraphUtilsV1.findByGuid(guid);
@@ -394,6 +417,40 @@ public final class EntityGraphRetriever {
return ret;
}
+ private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, AtlasAttribute attribute,
+ AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
+ Object ret = null;
+ AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
+ AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
+ AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
+ AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
+ AtlasRelationshipEndDef attributeEndDef = null;
+
+ if (endDef1Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef1.getName(), attribute.getName())) {
+ attributeEndDef = endDef1;
+
+ } else if (endDef2Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef2.getName(), attribute.getName())) {
+ attributeEndDef = endDef2;
+ }
+
+ String relationshipLabel = attribute.getRelationshipEdgeLabel();
+
+ switch (attributeEndDef.getCardinality()) {
+ case SINGLE:
+ ret = mapVertexToObjectId(entityVertex, relationshipLabel, null, entityExtInfo, attributeEndDef.getIsContainer());
+ break;
+
+ case LIST:
+ case SET:
+ ret = mapVertexToRelationshipArrayAttribute(entityVertex, (AtlasArrayType) attribute.getAttributeType(), relationshipLabel,
+ entityExtInfo, attributeEndDef.getIsContainer());
+ break;
+ }
+
+ return ret;
+ }
+
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName);
@@ -451,6 +508,40 @@ public final class EntityGraphRetriever {
return arrValues;
}
+ private List<Object> mapVertexToRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasArrayType arrayType,
+ String relationshipName, AtlasEntityExtInfo entityExtInfo,
+ boolean isContainer) throws AtlasBaseException {
+
+ Iterator<AtlasEdge> relationshipEdges = graphHelper.getBothEdgesByLabel(entityVertex, relationshipName);
+ AtlasType arrayElementType = arrayType.getElementType();
+ List<AtlasEdge> arrayElements = new ArrayList<>();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Mapping array attribute {} for vertex {}", arrayElementType.getTypeName(), entityVertex);
+ }
+
+ while (relationshipEdges.hasNext()) {
+ arrayElements.add(relationshipEdges.next());
+ }
+
+ if (CollectionUtils.isEmpty(arrayElements)) {
+ return null;
+ }
+
+ List arrValues = new ArrayList(arrayElements.size());
+
+ for (Object element : arrayElements) {
+ Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, relationshipName,
+ entityExtInfo, isContainer);
+
+ if (arrValue != null) {
+ arrValues.add(arrValue);
+ }
+ }
+
+ return arrValues;
+ }
+
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value, String edgeLabel,
AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
Object ret = null;
@@ -538,7 +629,11 @@ public final class EntityGraphRetriever {
}
if (GraphHelper.elementExists(edge)) {
- final AtlasVertex referenceVertex = edge.getInVertex();
+ AtlasVertex referenceVertex = edge.getInVertex();
+
+ if (StringUtils.equals(getIdFromVertex(referenceVertex), getIdFromVertex(entityVertex))) {
+ referenceVertex = edge.getOutVertex();
+ }
if (referenceVertex != null) {
if (entityExtInfo != null && isOwnedAttribute) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/8fe110c3/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
new file mode 100644
index 0000000..6770223
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v1;
+
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+
+import static org.mockito.Mockito.mock;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class AtlasRelationshipStoreV1Test {
+
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ DeleteHandlerV1 deleteHandler;
+
+ @Inject
+ EntityGraphMapper graphMapper;
+
+ AtlasEntityStore entityStore;
+ AtlasRelationshipStore relationshipStore;
+ AtlasEntityWithExtInfo dbEntity;
+ AtlasEntityWithExtInfo tblEntity;
+ AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ new GraphBackedSearchIndexer(typeRegistry);
+
+ AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
+ TestUtilsV2.defineHiveTypes() };
+
+ for (AtlasTypesDef typesDef : testTypesDefs) {
+ AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry);
+
+ if (!typesToCreate.isEmpty()) {
+ typeDefStore.createTypesDef(typesToCreate);
+ }
+ }
+
+ dbEntity = TestUtilsV2.createDBEntityV2();
+ tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
+ }
+
+ @AfterClass
+ public void clear() {
+ AtlasGraphProvider.cleanup();
+ }
+
+ @BeforeTest
+ public void init() throws Exception {
+ entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
+ relationshipStore = new AtlasRelationshipStoreV1(typeRegistry);
+
+ RequestContextV1.clear();
+ }
+
+ @Test
+ public void testDbTableRelationship() throws Exception {
+ // Add tests - in progress
+ }
+}
[39/39] atlas git commit: Merge branch 'master' into feature-odf
Posted by ma...@apache.org.
Merge branch 'master' into feature-odf
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/22780b93
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/22780b93
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/22780b93
Branch: refs/heads/feature-odf
Commit: 22780b933f88e50bff1915029fa7aa18d2a14960
Parents: cac4bee 7fb09b0
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Mon Jul 24 08:56:38 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jul 24 08:56:38 2017 -0700
----------------------------------------------------------------------
DISCLAIMER.txt | 19 +-
README.txt | 14 +-
addons/falcon-bridge-shim/pom.xml | 2 +-
addons/falcon-bridge/pom.xml | 2 +-
addons/hdfs-model/pom.xml | 2 +-
addons/hive-bridge-shim/pom.xml | 2 +-
addons/hive-bridge/pom.xml | 2 +-
addons/models/0010-base_model.json | 42 +-
addons/models/0030-hive_model.json | 102 +++-
addons/models/0050-falcon_model.json | 61 +-
addons/models/0060-hbase_model.json | 44 +-
addons/models/0080-storm_model.json | 23 +-
addons/sqoop-bridge-shim/pom.xml | 2 +-
addons/sqoop-bridge/pom.xml | 2 +-
addons/storm-bridge-shim/pom.xml | 2 +-
addons/storm-bridge/pom.xml | 2 +-
authorization/pom.xml | 2 +-
build-tools/pom.xml | 2 +-
catalog/pom.xml | 2 +-
client/pom.xml | 2 +-
common/pom.xml | 2 +-
.../org/apache/atlas/repository/Constants.java | 4 +-
.../java/org/apache/atlas/utils/MD5Utils.java | 66 --
.../org/apache/atlas/utils/SHA256Utils.java | 68 +++
dashboardv2/gruntfile.js | 3 +
dashboardv2/package.json | 57 +-
dashboardv2/pom.xml | 4 +-
dashboardv2/public/css/scss/main.scss | 2 +-
dashboardv2/public/css/scss/override.scss | 22 +-
dashboardv2/public/css/scss/tag.scss | 40 +-
dashboardv2/public/index.html | 1 +
dashboardv2/public/js/collection/VSearchList.js | 3 +-
dashboardv2/public/js/main.js | 4 +
dashboardv2/public/js/modules/Modal.js | 11 +-
dashboardv2/public/js/router/Router.js | 48 +-
.../business_catalog/BusinessCatalogHeader.html | 2 +-
.../public/js/templates/common/modal.html | 7 +-
.../search/AdvancedSearchInfo_tmpl.html | 2 +-
.../templates/search/SearchLayoutView_tmpl.html | 4 +-
.../search/SearchResultLayoutView_tmpl.html | 15 -
.../public/js/templates/site/header.html | 2 +-
.../public/js/utils/CommonViewFunction.js | 87 ++-
dashboardv2/public/js/utils/TableLayout.js | 43 +-
dashboardv2/public/js/utils/Utils.js | 17 +-
.../js/views/audit/AuditTableLayoutView.js | 4 +-
.../views/audit/CreateAuditTableLayoutView.js | 4 +-
.../views/business_catalog/SideNavLayoutView.js | 6 +-
.../views/detail_page/DetailPageLayoutView.js | 9 +-
.../views/entity/EntityDetailTableLayoutView.js | 11 +-
.../public/js/views/search/QueryBuilderView.js | 25 +-
.../js/views/search/SearchDetailLayoutView.js | 3 +-
.../public/js/views/search/SearchLayoutView.js | 257 ++++----
.../public/js/views/search/SearchQueryView.js | 21 +-
.../js/views/search/SearchResultLayoutView.js | 207 +++++--
.../js/views/tag/TagDetailTableLayoutView.js | 31 +-
.../public/js/views/tag/TagLayoutView.js | 4 +-
.../public/js/views/tag/addTagModalView.js | 13 +-
distro/pom.xml | 2 +-
distro/src/bin/atlas_stop.py | 27 +-
distro/src/conf/atlas-application.properties | 9 +-
docs/pom.xml | 8 +-
docs/src/site/site.xml | 14 +-
docs/src/site/twiki/Import-API-Options.twiki | 22 +
docs/src/site/twiki/Import-API.twiki | 6 +-
docs/src/site/twiki/InstallationSteps.twiki | 2 +-
docs/src/site/twiki/index.twiki | 8 -
graphdb/api/pom.xml | 2 +-
graphdb/common/pom.xml | 2 +-
graphdb/graphdb-impls/pom.xml | 2 +-
graphdb/pom.xml | 2 +-
graphdb/titan0/pom.xml | 2 +-
graphdb/titan1/pom.xml | 2 +-
intg/pom.xml | 2 +-
.../java/org/apache/atlas/AtlasErrorCode.java | 8 +-
.../model/discovery/AtlasSearchResult.java | 78 ++-
.../atlas/model/discovery/SearchParameters.java | 34 +-
.../atlas/model/impexp/AtlasImportRequest.java | 18 +-
.../atlas/model/instance/AtlasEntity.java | 38 +-
.../atlas/model/instance/AtlasRelationship.java | 21 +-
.../atlas/model/instance/AtlasStruct.java | 2 +-
.../model/typedef/AtlasRelationshipDef.java | 2 +
.../model/typedef/AtlasRelationshipEndDef.java | 25 +-
.../atlas/type/AtlasClassificationType.java | 30 +-
.../org/apache/atlas/type/AtlasEntityType.java | 125 +++-
.../atlas/type/AtlasRelationshipType.java | 82 ++-
.../org/apache/atlas/type/AtlasStructType.java | 126 +++-
.../java/org/apache/atlas/type/AtlasType.java | 3 +
.../apache/atlas/type/AtlasTypeRegistry.java | 4 +
.../apache/atlas/TestRelationshipUtilsV2.java | 267 ++++++++
.../test/java/org/apache/atlas/TestUtilsV2.java | 1 -
notification/pom.xml | 2 +-
.../apache/atlas/kafka/AtlasKafkaConsumer.java | 16 +-
.../apache/atlas/kafka/KafkaNotification.java | 12 +-
.../notification/NotificationConsumer.java | 8 +
.../apache/atlas/kafka/KafkaConsumerTest.java | 16 +-
.../atlas/kafka/KafkaNotificationTest.java | 2 +-
.../AbstractNotificationConsumerTest.java | 11 +-
plugin-classloader/pom.xml | 2 +-
pom.xml | 418 +++++++------
repository/pom.xml | 2 +-
.../atlas/discovery/AtlasDiscoveryService.java | 2 +-
.../ClassificationSearchProcessor.java | 211 +++++++
.../atlas/discovery/EntityDiscoveryService.java | 170 ++++--
.../atlas/discovery/EntitySearchProcessor.java | 232 +++++++
.../discovery/FullTextSearchProcessor.java | 168 +++++
.../org/apache/atlas/discovery/GremlinStep.java | 389 ------------
.../apache/atlas/discovery/SearchContext.java | 130 ++++
.../apache/atlas/discovery/SearchPipeline.java | 611 -------------------
.../apache/atlas/discovery/SearchProcessor.java | 398 ++++++++++++
.../org/apache/atlas/discovery/SolrStep.java | 288 ---------
.../atlas/repository/graph/GraphHelper.java | 141 ++++-
.../graph/TypedInstanceToGraphMapper.java | 4 +-
.../atlas/repository/impexp/ExportService.java | 26 +-
.../atlas/repository/impexp/ImportService.java | 2 +-
.../apache/atlas/repository/impexp/ZipSink.java | 21 +
.../atlas/repository/impexp/ZipSource.java | 2 +-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 23 +
.../store/graph/AtlasRelationshipStore.java | 7 +
.../store/graph/AtlasTypeDefGraphStore.java | 17 +
.../graph/v1/AtlasEntityGraphDiscoveryV1.java | 18 +-
.../store/graph/v1/AtlasEntityStoreV1.java | 119 +++-
.../store/graph/v1/AtlasGraphUtilsV1.java | 10 +-
.../graph/v1/AtlasRelationshipDefStoreV1.java | 7 +-
.../graph/v1/AtlasRelationshipStoreV1.java | 234 ++++---
.../store/graph/v1/DeleteHandlerV1.java | 33 +-
.../store/graph/v1/EntityGraphMapper.java | 446 ++++++++++++--
.../store/graph/v1/EntityGraphRetriever.java | 171 +++++-
.../org/apache/atlas/util/SearchTracker.java | 16 +-
.../test/java/org/apache/atlas/TestModules.java | 3 -
.../ReverseReferenceUpdateSoftDeleteTest.java | 1 +
.../repository/impexp/ImportServiceTest.java | 25 +-
.../impexp/ZipFileResourceTestUtils.java | 3 +
.../atlas/repository/impexp/ZipSinkTest.java | 79 ++-
.../AtlasRelationshipStoreHardDeleteV1Test.java | 54 ++
.../AtlasRelationshipStoreSoftDeleteV1Test.java | 55 ++
.../graph/v1/AtlasRelationshipStoreV1Test.java | 496 +++++++++++++++
.../services/EntityDiscoveryServiceTest.java | 12 +-
repository/src/test/resources/ctas.zip | Bin 0 -> 7674 bytes
server-api/pom.xml | 2 +-
shaded/hbase-client-shaded/pom.xml | 2 +-
shaded/hbase-server-shaded/pom.xml | 2 +-
typesystem/pom.xml | 2 +-
.../apache/atlas/typesystem/persistence/Id.java | 4 +-
.../persistence/ReferenceableInstance.java | 4 +-
.../typesystem/persistence/StructInstance.java | 4 +-
webapp/pom.xml | 4 +-
.../notification/NotificationHookConsumer.java | 58 +-
.../atlas/web/filters/ActiveServerFilter.java | 15 +-
.../web/filters/AtlasAuthenticationFilter.java | 25 +-
.../atlas/web/resources/AdminResource.java | 8 +-
.../apache/atlas/web/rest/DiscoveryREST.java | 2 +-
.../NotificationHookConsumerKafkaTest.java | 2 +-
.../web/filters/ActiveServerFilterTest.java | 35 +-
.../atlas/web/integration/BaseResourceIT.java | 2 +-
154 files changed, 5177 insertions(+), 2523 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/22780b93/pom.xml
----------------------------------------------------------------------
[33/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering (# 5)
Posted by ma...@apache.org.
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 5)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/62d85a4c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/62d85a4c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/62d85a4c
Branch: refs/heads/feature-odf
Commit: 62d85a4caebeb8f459c0a1774fade83cd49b9c35
Parents: 13ba156
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Thu Jul 20 15:42:46 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jul 20 21:28:13 2017 -0700
----------------------------------------------------------------------
.../atlas/discovery/EntitySearchProcessor.java | 2 --
.../discovery/FullTextSearchProcessor.java | 12 ++++------
.../apache/atlas/discovery/SearchProcessor.java | 23 +++++++++++++-------
3 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/62d85a4c/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index a3525c9..efcfb7f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -18,10 +18,8 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
http://git-wip-us.apache.org/repos/asf/atlas/blob/62d85a4c/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 1b19a0e..0d1c39b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -20,12 +20,10 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.utils.AtlasPerfTracer;
-import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,7 +53,8 @@ public class FullTextSearchProcessor extends SearchProcessor {
Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
- queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
+ queryString.append(AND_STR);
+ appendIndexQueryValue(typeAndSubTypeNames, queryString);
} else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
@@ -68,17 +67,14 @@ public class FullTextSearchProcessor extends SearchProcessor {
Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
- queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
+ queryString.append(AND_STR);
+ appendIndexQueryValue(typeAndSubTypeNames, queryString);
} else {
LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
context.getClassificationType().getTypeName(), typeAndSubTypeNames.size());
}
}
- if (context.getSearchParameters().getExcludeDeletedEntities()) {
- queryString.append(AND_STR).append("(ACTIVE)");
- }
-
queryString.append(")");
indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString());
http://git-wip-us.apache.org/repos/asf/atlas/blob/62d85a4c/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 7950127..2862165 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -48,12 +48,12 @@ public abstract class SearchProcessor {
public static final String AND_STR = " AND ";
public static final String EMPTY_STRING = "";
public static final String SPACE_STRING = " ";
- public static final String BRACE_OPEN_STR = "( ";
- public static final String BRACE_CLOSE_STR = " )";
+ public static final String BRACE_OPEN_STR = "(";
+ public static final String BRACE_CLOSE_STR = ")";
public static final char DOUBLE_QUOTE = '"';
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
- private static final char[] OFFENDING_CHARS = {'@', '/', ' '}; // This can grow as we discover corner cases
+ private static final char[] OFFENDING_CHARS = { '@', '/', ' ' }; // This can grow as we discover corner cases
static
{
@@ -182,16 +182,13 @@ public abstract class SearchProcessor {
}
protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
- String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
-
if (CollectionUtils.isNotEmpty(typeAndAllSubTypes)) {
if (solrQuery.length() > 0) {
solrQuery.append(AND_STR);
}
- solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": (")
- .append(typeAndSubtypesString)
- .append(")");
+ solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":");
+ appendIndexQueryValue(typeAndAllSubTypes, solrQuery);
}
}
@@ -392,6 +389,16 @@ public abstract class SearchProcessor {
return ret;
}
+ protected String appendIndexQueryValue(Set<String> values, StringBuilder sb) {
+ sb.append(BRACE_OPEN_STR);
+ for (String value : values) {
+ sb.append(escapeIndexQueryValue(value)).append(SPACE_STRING);
+ }
+ sb.append(BRACE_CLOSE_STR);
+
+ return sb.toString();
+ }
+
private static int getApplicationProperty(String propertyName, int defaultValue) {
try {
return ApplicationProperties.get().getInt(propertyName, defaultValue);
[10/39] atlas git commit: ATLAS-1949: Fix coverity scan issues and IT
failures due to ATLAS-1943
Posted by ma...@apache.org.
ATLAS-1949: Fix coverity scan issues and IT failures due to ATLAS-1943
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c3808cf1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c3808cf1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c3808cf1
Branch: refs/heads/feature-odf
Commit: c3808cf1f9010467801a0a1567d60a407fed4f79
Parents: 18745cf
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Jul 13 13:06:59 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jul 13 13:29:14 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/AtlasErrorCode.java | 1 +
.../atlas/repository/graph/GraphHelper.java | 4 ++++
.../store/graph/v1/EntityGraphMapper.java | 8 +++----
.../store/graph/v1/EntityGraphRetriever.java | 23 ++++++++++++++------
4 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3808cf1/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index fd1b004..b24f99f 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -88,6 +88,7 @@ public enum AtlasErrorCode {
RELATIONSHIPDEF_END1_NAME_INVALID(400, "ATLAS-400-00-041", "{0}: invalid end1 name. Name must not contain query keywords"),
RELATIONSHIPDEF_END2_NAME_INVALID(400, "ATLAS-400-00-042", "{0}: invalid end2 name. Name must not contain query keywords"),
RELATIONSHIPDEF_NOT_DEFINED(400, "ATLAS-400-00-043", "No relationshipDef defined between {0} and {1} on attribute: {2}"),
+ RELATIONSHIPDEF_INVALID(400, "ATLAS-400-00-044", "Invalid relationshipDef: {0}"),
// All Not found enums go here
TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"),
TYPE_GUID_NOT_FOUND(404, "ATLAS-404-00-002", "Given type guid {0} was invalid"),
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3808cf1/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index c5e32d8..6f6d74b 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -1269,6 +1269,10 @@ public final class GraphHelper {
}
}
+ if (ret == null) {
+ ret = relationshipTypes.get(0).getRelationshipDef();
+ }
+
} else {
//relationshipTypes will have at least one relationshipDef
ret = relationshipTypes.get(0).getRelationshipDef();
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3808cf1/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index 68f8370..157f8cd 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -328,7 +328,7 @@ public class EntityGraphMapper {
newEdge = mapObjectIdValueUsingRelationship(ctx, context);
- if (ctx.getAttribute().getInverseRefAttribute() != null) {
+ if (newEdge != null && ctx.getAttribute().getInverseRefAttribute() != null) {
// Update the inverse reference using relationship on the target entity
addInverseReference(ctx.getAttribute().getInverseRefAttribute(), newEdge);
}
@@ -380,13 +380,13 @@ public class EntityGraphMapper {
case ARRAY:
// Add edge ID to property value
List<String> elements = inverseVertex.getProperty(propertyName, List.class);
- if (elements == null) {
+ if (newEdge != null && elements == null) {
elements = new ArrayList<>();
elements.add(newEdge.getId().toString());
inverseVertex.setProperty(propertyName, elements);
}
else {
- if (!elements.contains(newEdge.getId().toString())) {
+ if (newEdge != null && !elements.contains(newEdge.getId().toString())) {
elements.add(newEdge.getId().toString());
inverseVertex.setProperty(propertyName, elements);
}
@@ -945,7 +945,7 @@ public class EntityGraphMapper {
String newEntityId = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
AtlasEdge ret = currentEdge;
- if (!currentEntityId.equals(newEntityId) && entityVertex != null) {
+ if (!currentEntityId.equals(newEntityId)) {
// create a new relationship edge to the new attribute vertex from the instance
String relationshipName = AtlasGraphUtilsV1.getTypeName(currentEdge);
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3808cf1/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index e919d1c..4ac00a3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -419,13 +419,18 @@ public final class EntityGraphRetriever {
private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, AtlasAttribute attribute,
AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
- Object ret = null;
- AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
- AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
- AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
- AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
- AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
- AtlasRelationshipEndDef attributeEndDef = null;
+ Object ret = null;
+ AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
+
+ if (relationshipDef == null) {
+ throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, "relationshipDef is null");
+ }
+
+ AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
+ AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
+ AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
+ AtlasRelationshipEndDef attributeEndDef = null;
if (endDef1Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef1.getName(), attribute.getName())) {
attributeEndDef = endDef1;
@@ -434,6 +439,10 @@ public final class EntityGraphRetriever {
attributeEndDef = endDef2;
}
+ if (attributeEndDef == null) {
+ throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, relationshipDef.toString());
+ }
+
String relationshipLabel = attribute.getRelationshipEdgeLabel();
switch (attributeEndDef.getCardinality()) {
[23/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering (# 2)
Posted by ma...@apache.org.
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 2)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/cfb6b84f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/cfb6b84f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/cfb6b84f
Branch: refs/heads/feature-odf
Commit: cfb6b84f41b05275db826bbd43e4c39145b6d2d5
Parents: 377fe19
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Tue Jul 18 13:27:46 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 18 16:59:02 2017 -0700
----------------------------------------------------------------------
.../atlas/discovery/EntitySearchProcessor.java | 30 +++++++++++-----
.../discovery/FullTextSearchProcessor.java | 36 ++++++++++++++++++--
.../apache/atlas/discovery/SearchContext.java | 9 +++--
.../apache/atlas/discovery/SearchProcessor.java | 4 +--
4 files changed, 61 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 3204ecf..50376ef 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
@@ -39,18 +40,21 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
- final AtlasEntityType entityType = context.getEntityType();
- final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
- final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- final Set<String> solrAttributes = new HashSet<>();
- final Set<String> gremlinAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
+ final AtlasEntityType entityType = context.getEntityType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ final Set<String> solrAttributes = new HashSet<>();
+ final Set<String> gremlinAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
+
+ final AtlasClassificationType classificationType = context.getClassificationType();
+ final boolean filterClassification = classificationType != null && !context.needClassificationProcessor();
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
- final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
- final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
+ final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+ final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false);
StringBuilder solrQuery = new StringBuilder();
@@ -82,6 +86,10 @@ public class EntitySearchProcessor extends SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
}
+ if (filterClassification) {
+ query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
+ }
+
graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) {
@@ -93,6 +101,10 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+ if (filterClassification) {
+ query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
+ }
+
filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities()) {
@@ -115,7 +127,7 @@ public class EntitySearchProcessor extends SearchProcessor {
}
try {
- int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
+ int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery == null)) ? context.getSearchParameters().getOffset() : 0;
int limit = context.getSearchParameters().getLimit();
int resultIdx = qryOffset;
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 4ddd642..83368c2 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -22,12 +22,14 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
public class FullTextSearchProcessor extends SearchProcessor {
@@ -40,9 +42,39 @@ public class FullTextSearchProcessor extends SearchProcessor {
super(context);
SearchParameters searchParameters = context.getSearchParameters();
- String queryString = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
+ StringBuilder queryString = new StringBuilder();
- indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString);
+ queryString.append("v.\"").append(Constants.ENTITY_TEXT_PROPERTY_KEY).append("\":(").append(searchParameters.getQuery());
+
+ // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
+ // processing (and rejection) by subsequent EntitySearchProcessor
+ if (context.getEntityType() != null) {
+ Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
+
+ if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
+ queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
+ } else {
+ LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
+ context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+ }
+ }
+
+ // if search includes classification criteria, adding a filter here can help avoid unnecessary
+ // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
+ if (context.getClassificationType() != null) {
+ Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
+
+ if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
+ queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
+ } else {
+ LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
+ context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+ }
+ }
+
+ queryString.append(")");
+
+ indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 8dd7667..929f8d0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -61,7 +61,6 @@ public class SearchContext {
if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this));
-
}
}
@@ -104,15 +103,15 @@ public class SearchContext {
return toString(new StringBuilder()).toString();
}
- public boolean needFullTextrocessor() {
+ boolean needFullTextrocessor() {
return StringUtils.isNotEmpty(searchParameters.getQuery());
}
- public boolean needClassificationProcessor() {
- return classificationType != null;
+ boolean needClassificationProcessor() {
+ return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
}
- public boolean needEntityProcessor() {
+ boolean needEntityProcessor() {
return entityType != null;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index ff0bd2e..596b43b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
- solrQuery.append("v.\"__typeName\": (")
+ solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": (")
.append(typeAndSubtypesString)
.append(")");
}
@@ -206,7 +206,7 @@ public abstract class SearchProcessor {
solrQuery.append(AND_STR);
}
- solrQuery.append("v.\"__state\":").append("ACTIVE");
+ solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
}
}
[28/39] atlas git commit: ATLAS-1959: Enhance relationship attributes
to support cardinality mappings
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index 157f8cd..1282be5 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -45,6 +45,7 @@ import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
@@ -63,7 +64,11 @@ import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DE
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.PARTIAL_UPDATE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
+import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.string;
+import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
@Component
public class EntityGraphMapper {
@@ -144,6 +149,8 @@ public class EntityGraphMapper {
mapAttributes(createdEntity, vertex, CREATE, context);
+ mapRelationshipAttributes(createdEntity, vertex, CREATE, context);
+
resp.addEntity(CREATE, constructHeader(createdEntity, entityType, vertex));
addClassifications(context, guid, createdEntity.getClassifications());
}
@@ -157,6 +164,8 @@ public class EntityGraphMapper {
mapAttributes(updatedEntity, vertex, UPDATE, context);
+ mapRelationshipAttributes(updatedEntity, vertex, UPDATE, context);
+
if (isPartialUpdate) {
resp.addEntity(PARTIAL_UPDATE, constructHeader(updatedEntity, entityType, vertex));
} else {
@@ -238,6 +247,7 @@ public class EntityGraphMapper {
mapAttribute(attribute, attrValue, vertex, op, context);
}
+
} else if (op.equals(UPDATE)) {
for (String attrName : struct.getAttributes().keySet()) {
AtlasAttribute attribute = structType.getAttribute(attrName);
@@ -260,6 +270,41 @@ public class EntityGraphMapper {
}
}
+ private void mapRelationshipAttributes(AtlasEntity entity, AtlasVertex vertex, EntityOperation op,
+ EntityMutationContext context) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> mapRelationshipAttributes({}, {})", op, entity.getTypeName());
+ }
+
+ if (MapUtils.isNotEmpty(entity.getRelationshipAttributes())) {
+ AtlasEntityType entityType = getEntityType(entity.getTypeName());
+
+ if (op.equals(CREATE)) {
+ for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
+ Object attrValue = entity.getRelationshipAttribute(attribute.getName());
+
+ mapAttribute(attribute, attrValue, vertex, op, context);
+ }
+
+ } else if (op.equals(UPDATE)) {
+ // relationship attributes mapping
+ for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
+ if (attribute != null && entity.hasRelationshipAttribute(attribute.getName())) {
+ Object attrValue = entity.getRelationshipAttribute(attribute.getName());
+
+ mapAttribute(attribute, attrValue, vertex, op, context);
+ }
+ }
+ }
+
+ updateModificationMetadata(vertex);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== mapRelationshipAttributes({}, {})", op, entity.getTypeName());
+ }
+ }
+
private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException {
if (attrValue == null) {
AtlasAttributeDef attributeDef = attribute.getAttributeDef();
@@ -309,14 +354,15 @@ public class EntityGraphMapper {
}
case OBJECT_ID_TYPE: {
- String edgeLabel = ctx.getAttribute().getRelationshipEdgeLabel();
+ String edgeLabel = ctx.getAttribute().getRelationshipEdgeLabel();
+ AtlasRelationshipEdgeDirection edgeDirection = ctx.getAttribute().getRelationshipEdgeDirection();
// legacy case - if relationship attribute doesn't exist, use legacy edge label.
if (StringUtils.isEmpty(edgeLabel)) {
edgeLabel = AtlasGraphUtilsV1.getEdgeLabel(ctx.getVertexProperty());
}
- AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel);
+ AtlasEdge currentEdge = graphHelper.getEdgeForLabel(ctx.getReferringVertex(), edgeLabel, edgeDirection);
AtlasEdge newEdge = null;
if (ctx.getValue() != null) {
@@ -328,14 +374,41 @@ public class EntityGraphMapper {
newEdge = mapObjectIdValueUsingRelationship(ctx, context);
- if (newEdge != null && ctx.getAttribute().getInverseRefAttribute() != null) {
+ // legacy case update inverse attribute
+ if (ctx.getAttribute().getInverseRefAttribute() != null) {
// Update the inverse reference using relationship on the target entity
addInverseReference(ctx.getAttribute().getInverseRefAttribute(), newEdge);
}
}
+ // created new relationship,
+ // record entity update on both vertices of the new relationship
+ if (currentEdge == null && newEdge != null) {
+
+ // based on relationship edge direction record update only on attribute vertex
+ if (edgeDirection == IN) {
+ recordEntityUpdate(newEdge.getOutVertex());
+
+ } else {
+ recordEntityUpdate(newEdge.getInVertex());
+ }
+ }
+
+ // update references, if current and new edge don't match
+ // record entity update on new reference and delete(edge) old reference.
if (currentEdge != null && !currentEdge.equals(newEdge)) {
- deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), ctx.getAttribute().isOwnedRef(), true);
+
+ //record entity update on new edge
+ if (isRelationshipEdge(newEdge)) {
+ AtlasVertex attrVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(ctx.getValue()));
+
+ recordEntityUpdate(attrVertex);
+ updateModificationMetadata(attrVertex);
+ }
+
+ //delete old reference
+ deleteHandler.deleteEdgeReference(currentEdge, ctx.getAttrType().getTypeCategory(), ctx.getAttribute().isOwnedRef(),
+ true, ctx.getAttribute().getRelationshipEdgeDirection());
}
return newEdge;
@@ -402,7 +475,7 @@ public class EntityGraphMapper {
if (inverseUpdated) {
updateModificationMetadata(inverseVertex);
- AtlasObjectId inverseEntityId = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(inverseVertex), inverseType.getTypeName());
+ AtlasObjectId inverseEntityId = new AtlasObjectId(getIdFromVertex(inverseVertex), inverseType.getTypeName());
RequestContextV1.get().recordEntityUpdate(inverseEntityId);
}
}
@@ -424,7 +497,7 @@ public class EntityGraphMapper {
if (entityType.hasRelationshipAttribute(inverseAttributeName)) {
String relationshipName = graphHelper.getRelationshipDefName(inverseVertex, entityType, inverseAttributeName);
- ret = getOrCreateRelationship(inverseVertex, vertex, relationshipName);
+ ret = getOrCreateRelationship(inverseVertex, vertex, relationshipName, inverseAttribute);
} else {
if (LOG.isDebugEnabled()) {
@@ -576,24 +649,47 @@ public class EntityGraphMapper {
String attributeName = ctx.getAttribute().getName();
AtlasType type = typeRegistry.getType(AtlasGraphUtilsV1.getTypeName(entityVertex));
+ AtlasRelationshipEdgeDirection edgeDirection = ctx.getAttribute().getRelationshipEdgeDirection();
+ String edgeLabel = ctx.getAttribute().getRelationshipEdgeLabel();
+
if (type instanceof AtlasEntityType) {
AtlasEntityType entityType = (AtlasEntityType) type;
// use relationship to create/update edges
if (entityType.hasRelationshipAttribute(attributeName)) {
if (ctx.getCurrentEdge() != null) {
- ret = updateRelationship(ctx.getCurrentEdge(), attributeVertex);
+ ret = updateRelationship(ctx.getCurrentEdge(), attributeVertex, edgeDirection, ctx.getAttribute());
+
+ recordEntityUpdate(attributeVertex);
} else {
- String relationshipName = graphHelper.getRelationshipDefName(entityVertex, entityType, attributeName);
- ret = getOrCreateRelationship(entityVertex, attributeVertex, relationshipName);
- }
+ String relationshipName = graphHelper.getRelationshipDefName(entityVertex, entityType, attributeName);
+ AtlasVertex fromVertex;
+ AtlasVertex toVertex;
+
+ if (edgeDirection == IN) {
+ fromVertex = attributeVertex;
+ toVertex = entityVertex;
+
+ } else {
+ fromVertex = entityVertex;
+ toVertex = attributeVertex;
+ }
+ boolean relationshipExists = isRelationshipExists(fromVertex, toVertex, edgeLabel);
+
+ ret = getOrCreateRelationship(fromVertex, toVertex, relationshipName, ctx.getAttribute());
+ // if relationship did not exist before and new relationship was created
+ // record entity update on both relationship vertices
+ if (!relationshipExists) {
+ recordEntityUpdate(attributeVertex);
+ }
+ }
} else {
// use legacy way to create/update edges
if (LOG.isDebugEnabled()) {
- LOG.debug("No RelationshipDef defined between {} and {} on attribute: {}", AtlasGraphUtilsV1.getTypeName(entityVertex),
- AtlasGraphUtilsV1.getTypeName(attributeVertex), attributeName);
+ LOG.debug("No RelationshipDef defined between {} and {} on attribute: {}", getTypeName(entityVertex),
+ getTypeName(attributeVertex), attributeName);
}
ret = mapObjectIdValue(ctx, context);
@@ -728,6 +824,21 @@ public class EntityGraphMapper {
return newElementsCreated;
}
+ private boolean isRelationshipAttribute(AtlasAttribute attribute) {
+ boolean ret = false;
+
+ if (attribute != null) {
+ AtlasStructType structType = attribute.getDefinedInType();
+ String attributeName = attribute.getName();
+
+ if (structType instanceof AtlasEntityType) {
+ ret = ((AtlasEntityType) structType).hasRelationshipAttribute(attributeName);
+ }
+ }
+
+ return ret;
+ }
+
private AtlasEdge createVertex(AtlasStruct struct, AtlasVertex referringVertex, String edgeLabel, EntityMutationContext context) throws AtlasBaseException {
AtlasVertex vertex = createStructVertex(struct);
@@ -767,6 +878,16 @@ public class EntityGraphMapper {
return (AtlasStructType)objType;
}
+ private AtlasEntityType getEntityType(String typeName) throws AtlasBaseException {
+ AtlasType objType = typeRegistry.getType(typeName);
+
+ if (!(objType instanceof AtlasEntityType)) {
+ throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, typeName);
+ }
+
+ return (AtlasEntityType)objType;
+ }
+
private Object mapCollectionElementsToVertex(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
switch(ctx.getAttrType().getTypeCategory()) {
case PRIMITIVE:
@@ -918,8 +1039,8 @@ public class EntityGraphMapper {
// Update edge if it exists
AtlasVertex currentVertex = currentEdge.getInVertex();
- String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentVertex);
- String newEntityId = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
+ String currentEntityId = getIdFromVertex(currentVertex);
+ String newEntityId = getIdFromVertex(entityVertex);
AtlasEdge newEdge = currentEdge;
if (!currentEntityId.equals(newEntityId)) {
@@ -936,16 +1057,25 @@ public class EntityGraphMapper {
return newEdge;
}
- private AtlasEdge updateRelationship(AtlasEdge currentEdge, final AtlasVertex entityVertex) throws AtlasBaseException {
+ private AtlasEdge updateRelationship(AtlasEdge currentEdge, final AtlasVertex newEntityVertex,
+ AtlasRelationshipEdgeDirection edgeDirection, AtlasAttribute attribute)
+ throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
- LOG.debug("Updating entity reference using relationship {} for reference attribute {}", AtlasGraphUtilsV1.getTypeName(entityVertex));
+ LOG.debug("Updating entity reference using relationship {} for reference attribute {}", getTypeName(newEntityVertex));
}
- String currentEntityId = AtlasGraphUtilsV1.getIdFromVertex(currentEdge.getInVertex());
- String newEntityId = AtlasGraphUtilsV1.getIdFromVertex(entityVertex);
- AtlasEdge ret = currentEdge;
+ // Max's manager updated from Jane to Julius (Max.manager --> Jane.subordinates)
+ // manager attribute (OUT direction), current manager vertex (Jane) (IN vertex)
- if (!currentEntityId.equals(newEntityId)) {
+ // Max's mentor updated from John to Jane (John.mentee --> Max.mentor)
+ // mentor attribute (IN direction), current mentee vertex (John) (OUT vertex)
+ String currentEntityId = (edgeDirection == IN) ? getIdFromVertex(currentEdge.getOutVertex()) :
+ getIdFromVertex(currentEdge.getInVertex());
+
+ String newEntityId = getIdFromVertex(newEntityVertex);
+ AtlasEdge ret = currentEdge;
+
+ if (!currentEntityId.equals(newEntityId) && newEntityVertex != null) {
// create a new relationship edge to the new attribute vertex from the instance
String relationshipName = AtlasGraphUtilsV1.getTypeName(currentEdge);
@@ -953,7 +1083,8 @@ public class EntityGraphMapper {
relationshipName = currentEdge.getLabel();
}
- ret = getOrCreateRelationship(currentEdge.getOutVertex(), entityVertex, relationshipName);
+ ret = (edgeDirection == IN) ? getOrCreateRelationship(newEntityVertex, currentEdge.getInVertex(), relationshipName, attribute) :
+ getOrCreateRelationship(currentEdge.getOutVertex(), newEntityVertex, relationshipName, attribute);
}
return ret;
@@ -983,8 +1114,7 @@ public class EntityGraphMapper {
//Removes unused edges from the old collection, compared to the new collection
private List<AtlasEdge> removeUnusedArrayEntries(AtlasAttribute attribute, List<AtlasEdge> currentEntries, List<AtlasEdge> newEntries) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(currentEntries)) {
- AtlasStructType entityType = attribute.getDefinedInType();
- AtlasType entryType = ((AtlasArrayType)attribute.getAttributeType()).getElementType();
+ AtlasType entryType = ((AtlasArrayType) attribute.getAttributeType()).getElementType();
if (AtlasGraphUtilsV1.isReference(entryType)) {
Collection<AtlasEdge> edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries);
@@ -993,7 +1123,8 @@ public class EntityGraphMapper {
List<AtlasEdge> additionalElements = new ArrayList<>();
for (AtlasEdge edge : edgesToRemove) {
- boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(), attribute.isOwnedRef(), true);
+ boolean deleted = deleteHandler.deleteEdgeReference(edge, entryType.getTypeCategory(), attribute.isOwnedRef(),
+ true, attribute.getRelationshipEdgeDirection());
if (!deleted) {
additionalElements.add(edge);
@@ -1021,7 +1152,7 @@ public class EntityGraphMapper {
private AtlasEntityHeader constructHeader(AtlasEntity entity, final AtlasEntityType type, AtlasVertex vertex) {
AtlasEntityHeader header = new AtlasEntityHeader(entity.getTypeName());
- header.setGuid(AtlasGraphUtilsV1.getIdFromVertex(vertex));
+ header.setGuid(getIdFromVertex(vertex));
for (AtlasAttribute attribute : type.getUniqAttributes().values()) {
header.setAttribute(attribute.getName(), entity.getAttribute(attribute.getName()));
@@ -1148,7 +1279,7 @@ public class EntityGraphMapper {
for (String classificationName : classificationNames) {
try {
- final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
+ final String entityTypeName = getTypeName(instanceVertex);
String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, classificationName);
AtlasEdge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
if (edge != null) {
@@ -1182,20 +1313,20 @@ public class EntityGraphMapper {
}
}
- private AtlasEdge getOrCreateRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName) throws AtlasBaseException {
- AtlasEdge ret = null;
- AtlasObjectId end1 = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(end1Vertex), AtlasGraphUtilsV1.getTypeName(end1Vertex));
- AtlasObjectId end2 = new AtlasObjectId(AtlasGraphUtilsV1.getIdFromVertex(end2Vertex), AtlasGraphUtilsV1.getTypeName(end2Vertex));
- AtlasRelationship relationship = relationshipStore.getOrCreate(new AtlasRelationship(relationshipName, end1, end2));
+ private AtlasEdge getOrCreateRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, String relationshipName, AtlasAttribute attribute) throws AtlasBaseException {
+ AtlasEdge ret = null;
+ AtlasObjectId end1 = new AtlasObjectId(getIdFromVertex(end1Vertex), AtlasGraphUtilsV1.getTypeName(end1Vertex));
+ AtlasObjectId end2 = new AtlasObjectId(getIdFromVertex(end2Vertex), AtlasGraphUtilsV1.getTypeName(end2Vertex));
+ AtlasRelationship relationship = relationshipStore.getOrCreate(new AtlasRelationship(relationshipName, end1, end2));
// return newly created AtlasEdge
- // if multiple edges are returned, compare using id to pick the right one
+ // if multiple edges are returned, compare using guid to pick the right one
Iterator<AtlasEdge> outEdges = graphHelper.getOutGoingEdgesByLabel(end1Vertex, relationship.getLabel());
while (outEdges.hasNext()) {
AtlasEdge edge = outEdges.next();
- if (AtlasGraphUtilsV1.getIdFromVertex(end2Vertex).equals(AtlasGraphUtilsV1.getIdFromVertex(edge.getInVertex()))) {
+ if (getIdFromVertex(end2Vertex).equals(getIdFromVertex(edge.getInVertex()))) {
ret = edge;
break;
}
@@ -1203,4 +1334,47 @@ public class EntityGraphMapper {
return ret;
}
+
+ private boolean isRelationshipExists(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
+ boolean ret = false;
+ Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(fromVertex, edgeLabel);
+
+ while (edges != null && edges.hasNext()) {
+ AtlasEdge edge = edges.next();
+ AtlasVertex inVertex = edge.getInVertex();
+
+ if (inVertex != null && StringUtils.equals(getIdFromVertex(inVertex), getIdFromVertex(toVertex))) {
+ ret = true;
+ }
+ }
+
+ return ret;
+ }
+
+ private void recordEntityUpdate(AtlasVertex vertex) {
+ AtlasObjectId objectId = new AtlasObjectId(GraphHelper.getGuid(vertex), GraphHelper.getTypeName(vertex));
+ RequestContextV1 req = RequestContextV1.get();
+
+ if (!objectIdsContain(req.getUpdatedEntityIds(), objectId)) {
+ req.recordEntityUpdate(objectId);
+ }
+ }
+
+ private boolean objectIdsContain(Collection<AtlasObjectId> objectIds, AtlasObjectId objectId) {
+ boolean ret = false;
+
+ if (objectIds != null && objectIds.isEmpty()) {
+ ret = false;
+
+ } else {
+ for (AtlasObjectId id : objectIds) {
+ if (StringUtils.equals(id.getGuid(), objectId.getGuid())) {
+ ret = true;
+ break;
+ }
+ }
+ }
+
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index f4257be..31fc837 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -69,6 +69,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_SHORT;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
public final class EntityGraphRetriever {
@@ -136,6 +137,10 @@ public final class EntityGraphRetriever {
return toAtlasEntityHeader(entityVertex, Collections.<String>emptySet());
}
+ public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex atlasVertex, Set<String> attributes) throws AtlasBaseException {
+ return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null;
+ }
+
private AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
AtlasVertex ret = AtlasGraphUtilsV1.findByGuid(guid);
@@ -188,7 +193,7 @@ public final class EntityGraphRetriever {
mapAttributes(entityVertex, entity, entityExtInfo);
- mapRelationshipAttributes(entityVertex, entity, entityExtInfo);
+ mapRelationshipAttributes(entityVertex, entity);
mapClassifications(entityVertex, entity, entityExtInfo);
}
@@ -300,23 +305,6 @@ public final class EntityGraphRetriever {
}
}
- private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
- AtlasType objType = typeRegistry.getType(entity.getTypeName());
-
- if (!(objType instanceof AtlasEntityType)) {
- throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, entity.getTypeName());
- }
-
- AtlasEntityType entityType = (AtlasEntityType) objType;
-
- for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
-
- Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attribute, entityExtInfo);
-
- entity.addRelationshipAttribute(attribute.getName(), attrValue);
- }
- }
-
public List<AtlasClassification> getClassifications(String guid) throws AtlasBaseException {
AtlasVertex instanceVertex = AtlasGraphUtilsV1.findByGuid(guid);
@@ -401,6 +389,7 @@ public final class EntityGraphRetriever {
String vertexPropertyName = attribute.getQualifiedName();
String edgeLabel = EDGE_LABEL_PREFIX + vertexPropertyName;
boolean isOwnedAttribute = attribute.isOwnedRef();
+ AtlasRelationshipEdgeDirection edgeDirection = attribute.getRelationshipEdgeDirection();
if (LOG.isDebugEnabled()) {
LOG.debug("Mapping vertex {} to atlas entity {}.{}", entityVertex, attribute.getDefinedInDef().getName(), attribute.getName());
@@ -417,13 +406,13 @@ public final class EntityGraphRetriever {
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo);
break;
case OBJECT_ID_TYPE:
- ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute);
+ ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
case ARRAY:
- ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute);
+ ret = mapVertexToArray(entityVertex, (AtlasArrayType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
case MAP:
- ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute);
+ ret = mapVertexToMap(entityVertex, (AtlasMapType) attrType, vertexPropertyName, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
case CLASSIFICATION:
// do nothing
@@ -433,51 +422,10 @@ public final class EntityGraphRetriever {
return ret;
}
- private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, AtlasAttribute attribute,
- AtlasEntityExtInfo entityExtInfo) throws AtlasBaseException {
- Object ret = null;
- AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
-
- if (relationshipDef == null) {
- throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, "relationshipDef is null");
- }
-
- AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
- AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
- AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
- AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
- AtlasRelationshipEndDef attributeEndDef = null;
-
- if (endDef1Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef1.getName(), attribute.getName())) {
- attributeEndDef = endDef1;
-
- } else if (endDef2Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef2.getName(), attribute.getName())) {
- attributeEndDef = endDef2;
- }
-
- if (attributeEndDef == null) {
- throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, relationshipDef.toString());
- }
-
- String relationshipLabel = attribute.getRelationshipEdgeLabel();
-
- switch (attributeEndDef.getCardinality()) {
- case SINGLE:
- ret = mapVertexToObjectId(entityVertex, relationshipLabel, null, entityExtInfo, attributeEndDef.getIsContainer());
- break;
-
- case LIST:
- case SET:
- ret = mapVertexToRelationshipArrayAttribute(entityVertex, (AtlasArrayType) attribute.getAttributeType(), relationshipLabel,
- entityExtInfo, attributeEndDef.getIsContainer());
- break;
- }
-
- return ret;
- }
-
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasMapType atlasMapType, final String propertyName,
- AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
+ AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
+ AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
+
List<String> mapKeys = GraphHelper.getListProperty(entityVertex, propertyName);
if (CollectionUtils.isEmpty(mapKeys)) {
@@ -496,7 +444,9 @@ public final class EntityGraphRetriever {
final String edgeLabel = EDGE_LABEL_PREFIX + keyPropertyName;
final Object keyValue = GraphHelper.getMapValueProperty(mapValueType, entityVertex, keyPropertyName);
- Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, edgeLabel, entityExtInfo, isOwnedAttribute);
+ Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, edgeLabel,
+ entityExtInfo, isOwnedAttribute, edgeDirection);
+
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
@@ -506,7 +456,9 @@ public final class EntityGraphRetriever {
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasArrayType arrayType, String propertyName,
- AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
+ AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
+ AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
+
AtlasType arrayElementType = arrayType.getElementType();
List<Object> arrayElements = GraphHelper.getArrayElementsProperty(arrayElementType, entityVertex, propertyName);
@@ -522,8 +474,8 @@ public final class EntityGraphRetriever {
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
for (Object element : arrayElements) {
- Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element,
- edgeLabel, entityExtInfo, isOwnedAttribute);
+ Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
+ entityExtInfo, isOwnedAttribute, edgeDirection);
if (arrValue != null) {
arrValues.add(arrValue);
@@ -533,42 +485,9 @@ public final class EntityGraphRetriever {
return arrValues;
}
- private List<Object> mapVertexToRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasArrayType arrayType,
- String relationshipName, AtlasEntityExtInfo entityExtInfo,
- boolean isContainer) throws AtlasBaseException {
-
- Iterator<AtlasEdge> relationshipEdges = graphHelper.getBothEdgesByLabel(entityVertex, relationshipName);
- AtlasType arrayElementType = arrayType.getElementType();
- List<AtlasEdge> arrayElements = new ArrayList<>();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Mapping array attribute {} for vertex {}", arrayElementType.getTypeName(), entityVertex);
- }
-
- while (relationshipEdges.hasNext()) {
- arrayElements.add(relationshipEdges.next());
- }
-
- if (CollectionUtils.isEmpty(arrayElements)) {
- return null;
- }
-
- List arrValues = new ArrayList(arrayElements.size());
-
- for (Object element : arrayElements) {
- Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, relationshipName,
- entityExtInfo, isContainer);
-
- if (arrValue != null) {
- arrValues.add(arrValue);
- }
- }
-
- return arrValues;
- }
-
- private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value, String edgeLabel,
- AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
+ private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value,
+ String edgeLabel, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
+ AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
Object ret = null;
switch (arrayElement.getTypeCategory()) {
@@ -587,7 +506,7 @@ public final class EntityGraphRetriever {
break;
case OBJECT_ID_TYPE:
- ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute);
+ ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
default:
@@ -646,11 +565,12 @@ public final class EntityGraphRetriever {
}
private AtlasObjectId mapVertexToObjectId(AtlasVertex entityVertex, String edgeLabel, AtlasEdge edge,
- AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute) throws AtlasBaseException {
+ AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
+ AtlasRelationshipEdgeDirection edgeDirection) throws AtlasBaseException {
AtlasObjectId ret = null;
if (edge == null) {
- edge = graphHelper.getEdgeForLabel(entityVertex, edgeLabel);
+ edge = graphHelper.getEdgeForLabel(entityVertex, edgeLabel, edgeDirection);
}
if (GraphHelper.elementExists(edge)) {
@@ -697,7 +617,102 @@ public final class EntityGraphRetriever {
return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null) : null;
}
- public AtlasEntityHeader toAtlasEntityHeader(AtlasVertex atlasVertex, Set<String> attributes) throws AtlasBaseException {
- return atlasVertex != null ? mapVertexToAtlasEntityHeader(atlasVertex, attributes) : null;
+ private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException {
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+
+ if (entityType == null) {
+ throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, entity.getTypeName());
+ }
+
+ for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
+ Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attribute);
+
+ entity.setRelationshipAttribute(attribute.getName(), attrValue);
+ }
+ }
+
+ private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, AtlasAttribute attribute) throws AtlasBaseException {
+ Object ret = null;
+ AtlasRelationshipDef relationshipDef = graphHelper.getRelationshipDef(entityVertex, entityType, attribute.getName());
+
+ if (relationshipDef == null) {
+ throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, "relationshipDef is null");
+ }
+
+ AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
+ AtlasEntityType endDef1Type = typeRegistry.getEntityTypeByName(endDef1.getType());
+ AtlasEntityType endDef2Type = typeRegistry.getEntityTypeByName(endDef2.getType());
+ AtlasRelationshipEndDef attributeEndDef = null;
+
+ if (endDef1Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef1.getName(), attribute.getName())) {
+ attributeEndDef = endDef1;
+ } else if (endDef2Type.isTypeOrSuperTypeOf(entityType.getTypeName()) && StringUtils.equals(endDef2.getName(), attribute.getName())) {
+ attributeEndDef = endDef2;
+ }
+
+ if (attributeEndDef == null) {
+ throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, relationshipDef.toString());
+ }
+
+ switch (attributeEndDef.getCardinality()) {
+ case SINGLE:
+ ret = mapRelatedVertexToObjectId(entityVertex, attribute);
+ break;
+
+ case LIST:
+ case SET:
+ ret = mapRelationshipArrayAttribute(entityVertex, attribute);
+ break;
+ }
+
+ return ret;
+ }
+
+ private AtlasObjectId mapRelatedVertexToObjectId(AtlasVertex entityVertex, AtlasAttribute attribute) throws AtlasBaseException {
+ AtlasEdge edge = graphHelper.getEdgeForLabel(entityVertex, attribute.getRelationshipEdgeLabel(), attribute.getRelationshipEdgeDirection());
+
+ return mapRelatedVertexToObjectId(entityVertex, edge);
+ }
+
+ private List<AtlasObjectId> mapRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasAttribute attribute) throws AtlasBaseException {
+ List<AtlasObjectId> ret = new ArrayList<>();
+ Iterator<AtlasEdge> edges = null;
+
+ if (attribute.getRelationshipEdgeDirection() == AtlasRelationshipEdgeDirection.IN) {
+ edges = graphHelper.getIncomingEdgesByLabel(entityVertex, attribute.getRelationshipEdgeLabel());
+ } else if (attribute.getRelationshipEdgeDirection() == AtlasRelationshipEdgeDirection.OUT) {
+ edges = graphHelper.getOutGoingEdgesByLabel(entityVertex, attribute.getRelationshipEdgeLabel());
+ }
+
+ if (edges != null) {
+ while (edges.hasNext()) {
+ AtlasEdge relationshipEdge = edges.next();
+
+ AtlasObjectId objectId = mapRelatedVertexToObjectId(entityVertex, relationshipEdge);
+
+ ret.add(objectId);
+ }
+ }
+
+ return ret;
+ }
+
+ private AtlasObjectId mapRelatedVertexToObjectId(AtlasVertex entityVertex, AtlasEdge edge) throws AtlasBaseException {
+ AtlasObjectId ret = null;
+
+ if (GraphHelper.elementExists(edge)) {
+ AtlasVertex referenceVertex = edge.getInVertex();
+
+ if (StringUtils.equals(getIdFromVertex(referenceVertex), getIdFromVertex(entityVertex))) {
+ referenceVertex = edge.getOutVertex();
+ }
+
+ if (referenceVertex != null) {
+ ret = new AtlasObjectId(GraphHelper.getGuid(referenceVertex), GraphHelper.getTypeName(referenceVertex));
+ }
+ }
+
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index 404225c..7210799 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.impexp;
import com.google.inject.Inject;
+import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index d901731..82692cf 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.impexp;
import com.google.common.collect.Sets;
+import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.impexp.AtlasImportRequest;
@@ -149,6 +150,8 @@ public class ZipFileResourceTestUtils {
AtlasExportResult exportResult = zipSource.getExportResult();
List<String> creationOrder = zipSource.getCreationOrder();
+ RequestContextV1.clear();
+
AtlasImportRequest request = getDefaultImportRequest();
AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java
new file mode 100644
index 0000000..2c31140
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreHardDeleteV1Test.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v1;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.testng.annotations.Guice;
+
+import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
+
+/**
+ * Inverse reference update test with {@link HardDeleteHandlerV1}
+ */
+@Guice(modules = TestModules.HardDeleteModule.class)
+public class AtlasRelationshipStoreHardDeleteV1Test extends AtlasRelationshipStoreV1Test {
+
+ @Override
+ protected void verifyRelationshipAttributeUpdate_NonComposite_OneToMany(AtlasEntity jane) throws Exception {
+ // Max should have been removed from the subordinates list, leaving only John.
+ verifyRelationshipAttributeList(jane, "subordinates", ImmutableList.of(employeeNameIdMap.get("John")));
+ }
+
+ @Override
+ protected void verifyRelationshipAttributeUpdate_NonComposite_ManyToOne(AtlasEntity a1, AtlasEntity a2,
+ AtlasEntity a3, AtlasEntity b) {
+
+ verifyRelationshipAttributeValue(a1, "oneB", null);
+
+ verifyRelationshipAttributeValue(a2, "oneB", null);
+
+ verifyRelationshipAttributeList(b, "manyA", ImmutableList.of(getAtlasObjectId(a3)));
+ }
+
+ @Override
+ protected void verifyRelationshipAttributeUpdate_NonComposite_OneToOne(AtlasEntity a1, AtlasEntity b) {
+ verifyRelationshipAttributeValue(a1, "b", null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java
new file mode 100644
index 0000000..33ef8c0
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreSoftDeleteV1Test.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v1;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.testng.annotations.Guice;
+
+import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
+
+
+/**
+ * Inverse reference update test with {@link SoftDeleteHandlerV1}
+ */
+@Guice(modules = TestModules.SoftDeleteModule.class)
+public class AtlasRelationshipStoreSoftDeleteV1Test extends AtlasRelationshipStoreV1Test {
+
+ @Override
+ protected void verifyRelationshipAttributeUpdate_NonComposite_OneToMany(AtlasEntity jane) throws Exception {
+ // Max is still in the subordinates list, as the edge still exists with state DELETED
+ verifyRelationshipAttributeList(jane, "subordinates", ImmutableList.of(employeeNameIdMap.get("John"), employeeNameIdMap.get("Max")));
+ }
+
+ @Override
+ protected void verifyRelationshipAttributeUpdate_NonComposite_ManyToOne(AtlasEntity a1, AtlasEntity a2,
+ AtlasEntity a3, AtlasEntity b) {
+
+ verifyRelationshipAttributeValue(a1, "oneB", b.getGuid());
+
+ verifyRelationshipAttributeValue(a2, "oneB", b.getGuid());
+
+ verifyRelationshipAttributeList(b, "manyA", ImmutableList.of(getAtlasObjectId(a1), getAtlasObjectId(a2), getAtlasObjectId(a3)));
+ }
+
+ @Override
+ protected void verifyRelationshipAttributeUpdate_NonComposite_OneToOne(AtlasEntity a1, AtlasEntity b) {
+ verifyRelationshipAttributeValue(a1, "b", b.getGuid());
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
index 6770223..31efe86 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
@@ -17,18 +17,25 @@
*/
package org.apache.atlas.repository.store.graph.v1;
+import com.google.common.collect.ImmutableList;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
-import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeTest;
@@ -36,11 +43,25 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import static org.apache.atlas.TestRelationshipUtilsV2.EMPLOYEE_TYPE;
+import static org.apache.atlas.TestRelationshipUtilsV2.getDepartmentEmployeeInstances;
+import static org.apache.atlas.TestRelationshipUtilsV2.getDepartmentEmployeeTypes;
+import static org.apache.atlas.TestRelationshipUtilsV2.getInverseReferenceTestTypes;
+import static org.apache.atlas.TestUtils.NAME;
+import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class AtlasRelationshipStoreV1Test {
+public abstract class AtlasRelationshipStoreV1Test {
@Inject
AtlasTypeRegistry typeRegistry;
@@ -56,32 +77,28 @@ public class AtlasRelationshipStoreV1Test {
AtlasEntityStore entityStore;
AtlasRelationshipStore relationshipStore;
- AtlasEntityWithExtInfo dbEntity;
- AtlasEntityWithExtInfo tblEntity;
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
+ protected Map<String, AtlasObjectId> employeeNameIdMap = new HashMap<>();
+
@BeforeClass
public void setUp() throws Exception {
new GraphBackedSearchIndexer(typeRegistry);
- AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(),
- TestUtilsV2.defineHiveTypes() };
+ // create employee relationship types
+ AtlasTypesDef employeeTypes = getDepartmentEmployeeTypes();
+ typeDefStore.createTypesDef(employeeTypes);
- for (AtlasTypesDef typesDef : testTypesDefs) {
- AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry);
+ AtlasEntitiesWithExtInfo employeeInstances = getDepartmentEmployeeInstances();
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(employeeInstances), false);
- if (!typesToCreate.isEmpty()) {
- typeDefStore.createTypesDef(typesToCreate);
- }
+ for (AtlasEntityHeader entityHeader : response.getCreatedEntities()) {
+ employeeNameIdMap.put((String) entityHeader.getAttribute(NAME), getAtlasObjectId(entityHeader));
}
- dbEntity = TestUtilsV2.createDBEntityV2();
- tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity());
- }
-
- @AfterClass
- public void clear() {
- AtlasGraphProvider.cleanup();
+ init();
+ AtlasTypesDef testTypes = getInverseReferenceTestTypes();
+ typeDefStore.createTypesDef(testTypes);
}
@BeforeTest
@@ -92,8 +109,384 @@ public class AtlasRelationshipStoreV1Test {
RequestContextV1.clear();
}
+ @AfterClass
+ public void clear() {
+ AtlasGraphProvider.cleanup();
+ }
+
+ @Test
+ public void testDepartmentEmployeeEntitiesUsingRelationship() throws Exception {
+ AtlasObjectId hrId = employeeNameIdMap.get("hr");
+ AtlasObjectId maxId = employeeNameIdMap.get("Max");
+ AtlasObjectId johnId = employeeNameIdMap.get("John");
+ AtlasObjectId juliusId = employeeNameIdMap.get("Julius");
+ AtlasObjectId janeId = employeeNameIdMap.get("Jane");
+
+ AtlasEntity hrDept = getEntityFromStore(hrId.getGuid());
+ AtlasEntity max = getEntityFromStore(maxId.getGuid());
+ AtlasEntity john = getEntityFromStore(johnId.getGuid());
+ AtlasEntity julius = getEntityFromStore(juliusId.getGuid());
+ AtlasEntity jane = getEntityFromStore(janeId.getGuid());
+
+ // Department relationship attributes
+ List<AtlasObjectId> deptEmployees = toAtlasObjectIds(hrDept.getRelationshipAttribute("employees"));
+ assertNotNull(deptEmployees);
+ assertEquals(deptEmployees.size(), 4);
+ assertObjectIdsContains(deptEmployees, maxId);
+ assertObjectIdsContains(deptEmployees, johnId);
+ assertObjectIdsContains(deptEmployees, juliusId);
+ assertObjectIdsContains(deptEmployees, janeId);
+
+ // Max employee validation
+ AtlasObjectId maxDepartmentId = toAtlasObjectId(max.getRelationshipAttribute("department"));
+ assertNotNull(maxDepartmentId);
+ assertObjectIdEquals(maxDepartmentId, hrId);
+
+ AtlasObjectId maxManagerId = toAtlasObjectId(max.getRelationshipAttribute("manager"));
+ assertNotNull(maxManagerId);
+ assertObjectIdEquals(maxManagerId, janeId);
+
+ AtlasObjectId maxMentorId = toAtlasObjectId(max.getRelationshipAttribute("mentor"));
+ assertNotNull(maxMentorId);
+ assertObjectIdEquals(maxMentorId, juliusId);
+
+ List<AtlasObjectId> maxMenteesId = toAtlasObjectIds(max.getRelationshipAttribute("mentees"));
+ assertNotNull(maxMenteesId);
+ assertEquals(maxMenteesId.size(), 1);
+ assertObjectIdEquals(maxMenteesId.get(0), johnId);
+
+ // John Employee validation
+ AtlasObjectId johnDepartmentId = toAtlasObjectId(john.getRelationshipAttribute("department"));
+ assertNotNull(johnDepartmentId);
+ assertObjectIdEquals(johnDepartmentId, hrId);
+
+ AtlasObjectId johnManagerId = toAtlasObjectId(john.getRelationshipAttribute("manager"));
+ assertNotNull(johnManagerId);
+ assertObjectIdEquals(johnManagerId, janeId);
+
+ AtlasObjectId johnMentorId = toAtlasObjectId(john.getRelationshipAttribute("mentor"));
+ assertNotNull(johnMentorId);
+ assertObjectIdEquals(johnMentorId, maxId);
+
+ List<AtlasObjectId> johnMenteesId = toAtlasObjectIds(john.getRelationshipAttribute("mentees"));
+ assertNull(johnMenteesId);
+
+ // Jane Manager validation
+ AtlasObjectId janeDepartmentId = toAtlasObjectId(jane.getRelationshipAttribute("department"));
+ assertNotNull(janeDepartmentId);
+ assertObjectIdEquals(janeDepartmentId, hrId);
+
+ AtlasObjectId janeManagerId = toAtlasObjectId(jane.getRelationshipAttribute("manager"));
+ assertNull(janeManagerId);
+
+ AtlasObjectId janeMentorId = toAtlasObjectId(jane.getRelationshipAttribute("mentor"));
+ assertNull(janeMentorId);
+
+ List<AtlasObjectId> janeMenteesId = toAtlasObjectIds(jane.getRelationshipAttribute("mentees"));
+ assertNull(janeMenteesId);
+
+ List<AtlasObjectId> janeSubordinateIds = toAtlasObjectIds(jane.getRelationshipAttribute("subordinates"));
+ assertNotNull(janeSubordinateIds);
+ assertEquals(janeSubordinateIds.size(), 2);
+ assertObjectIdsContains(janeSubordinateIds, maxId);
+ assertObjectIdsContains(janeSubordinateIds, johnId);
+
+ // Julius Manager validation
+ AtlasObjectId juliusDepartmentId = toAtlasObjectId(julius.getRelationshipAttribute("department"));
+ assertNotNull(juliusDepartmentId);
+ assertObjectIdEquals(juliusDepartmentId, hrId);
+
+ AtlasObjectId juliusManagerId = toAtlasObjectId(julius.getRelationshipAttribute("manager"));
+ assertNull(juliusManagerId);
+
+ AtlasObjectId juliusMentorId = toAtlasObjectId(julius.getRelationshipAttribute("mentor"));
+ assertNull(juliusMentorId);
+
+ List<AtlasObjectId> juliusMenteesId = toAtlasObjectIds(julius.getRelationshipAttribute("mentees"));
+ assertNotNull(juliusMenteesId);
+ assertEquals(juliusMenteesId.size(), 1);
+ assertObjectIdsContains(juliusMenteesId, maxId);
+
+ List<AtlasObjectId> juliusSubordinateIds = toAtlasObjectIds(julius.getRelationshipAttribute("subordinates"));
+ assertNull(juliusSubordinateIds);
+ }
+
+ @Test
+ public void testRelationshipAttributeUpdate_NonComposite_OneToMany() throws Exception {
+ AtlasObjectId maxId = employeeNameIdMap.get("Max");
+ AtlasObjectId juliusId = employeeNameIdMap.get("Julius");
+ AtlasObjectId janeId = employeeNameIdMap.get("Jane");
+
+ // Change Max's Employee.manager reference to Julius and apply the change as a partial update.
+ // This should also update Julius to add Max to the inverse Manager.subordinates reference.
+ AtlasEntity maxEntityForUpdate = new AtlasEntity(EMPLOYEE_TYPE);
+ maxEntityForUpdate.setRelationshipAttribute("manager", juliusId);
+
+ AtlasEntityType employeeType = typeRegistry.getEntityTypeByName(EMPLOYEE_TYPE);
+ Map<String, Object> uniqAttributes = Collections.<String, Object>singletonMap("name", "Max");
+ EntityMutationResponse updateResponse = entityStore.updateByUniqueAttributes(employeeType, uniqAttributes , new AtlasEntityWithExtInfo(maxEntityForUpdate));
+
+ List<AtlasEntityHeader> partialUpdatedEntities = updateResponse.getPartialUpdatedEntities();
+ assertEquals(partialUpdatedEntities.size(), 3);
+ // 3 entities should have been updated:
+ // * Max to change the Employee.manager reference
+ // * Julius to add Max to Manager.subordinates
+ // * Jane to remove Max from Manager.subordinates
+
+ AtlasEntitiesWithExtInfo updatedEntities = entityStore.getByIds(ImmutableList.of(maxId.getGuid(), juliusId.getGuid(), janeId.getGuid()));
+
+ // Max's manager updated as Julius
+ AtlasEntity maxEntity = updatedEntities.getEntity(maxId.getGuid());
+ verifyRelationshipAttributeValue(maxEntity, "manager", juliusId.getGuid());
+
+ // Max added to the subordinate list of Julius
+ AtlasEntity juliusEntity = updatedEntities.getEntity(juliusId.getGuid());
+ verifyRelationshipAttributeList(juliusEntity, "subordinates", ImmutableList.of(maxId));
+
+ // Max removed from the subordinate list of Julius
+ AtlasEntity janeEntity = updatedEntities.getEntity(janeId.getGuid());
+
+ // Jane's subordinates list includes John and Max for soft delete
+ // Jane's subordinates list includes only John for hard delete
+ verifyRelationshipAttributeUpdate_NonComposite_OneToMany(janeEntity);
+ }
+
+ @Test
+ public void testRelationshipAttributeUpdate_NonComposite_ManyToOne() throws Exception {
+ AtlasEntity a1 = new AtlasEntity("A");
+ a1.setAttribute(NAME, "a1_name");
+
+ AtlasEntity a2 = new AtlasEntity("A");
+ a2.setAttribute(NAME, "a2_name");
+
+ AtlasEntity a3 = new AtlasEntity("A");
+ a3.setAttribute(NAME, "a3_name");
+
+ AtlasEntity b = new AtlasEntity("B");
+ b.setAttribute(NAME, "b_name");
+
+ AtlasEntitiesWithExtInfo entitiesWithExtInfo = new AtlasEntitiesWithExtInfo();
+ entitiesWithExtInfo.addEntity(a1);
+ entitiesWithExtInfo.addEntity(a2);
+ entitiesWithExtInfo.addEntity(a3);
+ entitiesWithExtInfo.addEntity(b);
+ entityStore.createOrUpdate(new AtlasEntityStream(entitiesWithExtInfo) , false);
+
+ AtlasEntity bPartialUpdate = new AtlasEntity("B");
+ bPartialUpdate.setRelationshipAttribute("manyA", ImmutableList.of(getAtlasObjectId(a1), getAtlasObjectId(a2)));
+
+ init();
+ EntityMutationResponse response = entityStore.updateByUniqueAttributes(typeRegistry.getEntityTypeByName("B"),
+ Collections.singletonMap(NAME, b.getAttribute(NAME)),
+ new AtlasEntityWithExtInfo(bPartialUpdate));
+ // Verify 3 entities were updated:
+ // * set b.manyA reference to a1 and a2
+ // * set inverse a1.oneB reference to b
+ // * set inverse a2.oneB reference to b
+ assertEquals(response.getPartialUpdatedEntities().size(), 3);
+ AtlasEntitiesWithExtInfo updatedEntities = entityStore.getByIds(ImmutableList.of(a1.getGuid(), a2.getGuid(), b.getGuid()));
+
+ AtlasEntity a1Entity = updatedEntities.getEntity(a1.getGuid());
+ verifyRelationshipAttributeValue(a1Entity, "oneB", b.getGuid());
+
+ AtlasEntity a2Entity = updatedEntities.getEntity(a2.getGuid());
+ verifyRelationshipAttributeValue(a2Entity, "oneB", b.getGuid());
+
+ AtlasEntity bEntity = updatedEntities.getEntity(b.getGuid());
+ verifyRelationshipAttributeList(bEntity, "manyA", ImmutableList.of(getAtlasObjectId(a1), getAtlasObjectId(a2)));
+
+
+ bPartialUpdate.setRelationshipAttribute("manyA", ImmutableList.of(getAtlasObjectId(a3)));
+ init();
+ response = entityStore.updateByUniqueAttributes(typeRegistry.getEntityTypeByName("B"),
+ Collections.singletonMap(NAME, b.getAttribute(NAME)),
+ new AtlasEntityWithExtInfo(bPartialUpdate));
+ // Verify 4 entities were updated:
+ // * set b.manyA reference to a3
+ // * set inverse a3.oneB reference to b
+ // * disconnect inverse a1.oneB reference to b
+ // * disconnect inverse a2.oneB reference to b
+ assertEquals(response.getPartialUpdatedEntities().size(), 4);
+ init();
+
+ updatedEntities = entityStore.getByIds(ImmutableList.of(a1.getGuid(), a2.getGuid(), a3.getGuid(), b.getGuid()));
+ a1Entity = updatedEntities.getEntity(a1.getGuid());
+ a2Entity = updatedEntities.getEntity(a2.getGuid());
+ bEntity = updatedEntities.getEntity(b.getGuid());
+
+ AtlasEntity a3Entity = updatedEntities.getEntity(a3.getGuid());
+ verifyRelationshipAttributeValue(a3Entity, "oneB", b.getGuid());
+
+ verifyRelationshipAttributeUpdate_NonComposite_ManyToOne(a1Entity, a2Entity, a3Entity, bEntity);
+ }
+
+ @Test
+ public void testRelationshipAttributeUpdate_NonComposite_OneToOne() throws Exception {
+ AtlasEntity a1 = new AtlasEntity("A");
+ a1.setAttribute(NAME, "a1_name");
+
+ AtlasEntity a2 = new AtlasEntity("A");
+ a2.setAttribute(NAME, "a2_name");
+
+ AtlasEntity b = new AtlasEntity("B");
+ b.setAttribute(NAME, "b_name");
+
+ AtlasEntitiesWithExtInfo entitiesWithExtInfo = new AtlasEntitiesWithExtInfo();
+ entitiesWithExtInfo.addEntity(a1);
+ entitiesWithExtInfo.addEntity(a2);
+ entitiesWithExtInfo.addEntity(b);
+
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesWithExtInfo) , false);
+
+ AtlasEntity partialUpdateB = new AtlasEntity("B");
+ partialUpdateB.setRelationshipAttribute("a", getAtlasObjectId(a1));
+
+ init();
+ AtlasEntityType bType = typeRegistry.getEntityTypeByName("B");
+
+ response = entityStore.updateByUniqueAttributes(bType, Collections.singletonMap(NAME, b.getAttribute(NAME)), new AtlasEntityWithExtInfo(partialUpdateB));
+ List<AtlasEntityHeader> partialUpdatedEntitiesHeader = response.getPartialUpdatedEntities();
+ // Verify 2 entities were updated:
+ // * set b.a reference to a1
+ // * set inverse a1.b reference to b
+ assertEquals(partialUpdatedEntitiesHeader.size(), 2);
+ AtlasEntitiesWithExtInfo partialUpdatedEntities = entityStore.getByIds(ImmutableList.of(a1.getGuid(), b.getGuid()));
+
+ AtlasEntity a1Entity = partialUpdatedEntities.getEntity(a1.getGuid());
+ verifyRelationshipAttributeValue(a1Entity, "b", b.getGuid());
+
+ AtlasEntity bEntity = partialUpdatedEntities.getEntity(b.getGuid());
+ verifyRelationshipAttributeValue(bEntity, "a", a1.getGuid());
+
+ init();
+
+ // Update b.a to reference a2.
+ partialUpdateB.setRelationshipAttribute("a", getAtlasObjectId(a2));
+ response = entityStore.updateByUniqueAttributes(bType, Collections.<String, Object>singletonMap(NAME, b.getAttribute(NAME)), new AtlasEntityWithExtInfo(partialUpdateB));
+ partialUpdatedEntitiesHeader = response.getPartialUpdatedEntities();
+ // Verify 3 entities were updated:
+ // * set b.a reference to a2
+ // * set a2.b reference to b
+ // * disconnect a1.b reference
+ assertEquals(partialUpdatedEntitiesHeader.size(), 3);
+ partialUpdatedEntities = entityStore.getByIds(ImmutableList.of(a1.getGuid(), a2.getGuid(), b.getGuid()));
+
+ bEntity = partialUpdatedEntities.getEntity(b.getGuid());
+ verifyRelationshipAttributeValue(bEntity, "a", a2.getGuid());
+
+ AtlasEntity a2Entity = partialUpdatedEntities.getEntity(a2.getGuid());
+ verifyRelationshipAttributeValue(a2Entity, "b", b.getGuid());
+
+ a1Entity = partialUpdatedEntities.getEntity(a1.getGuid());
+ verifyRelationshipAttributeUpdate_NonComposite_OneToOne(a1Entity, bEntity);
+ }
+
@Test
- public void testDbTableRelationship() throws Exception {
- // Add tests - in progress
+ public void testRelationshipAttributeUpdate_NonComposite_ManyToMany() throws Exception {
+ AtlasEntity a1 = new AtlasEntity("A");
+ a1.setAttribute(NAME, "a1_name");
+
+ AtlasEntity a2 = new AtlasEntity("A");
+ a2.setAttribute(NAME, "a2_name");
+
+ AtlasEntity a3 = new AtlasEntity("A");
+ a3.setAttribute(NAME, "a3_name");
+
+ AtlasEntity b1 = new AtlasEntity("B");
+ b1.setAttribute(NAME, "b1_name");
+
+ AtlasEntity b2 = new AtlasEntity("B");
+ b2.setAttribute(NAME, "b2_name");
+
+ AtlasEntitiesWithExtInfo entitiesWithExtInfo = new AtlasEntitiesWithExtInfo();
+ entitiesWithExtInfo.addEntity(a1);
+ entitiesWithExtInfo.addEntity(a2);
+ entitiesWithExtInfo.addEntity(a3);
+ entitiesWithExtInfo.addEntity(b1);
+ entitiesWithExtInfo.addEntity(b2);
+ entityStore.createOrUpdate(new AtlasEntityStream(entitiesWithExtInfo) , false);
+
+ AtlasEntity b1PartialUpdate = new AtlasEntity("B");
+ b1PartialUpdate.setRelationshipAttribute("manyToManyA", ImmutableList.of(getAtlasObjectId(a1), getAtlasObjectId(a2)));
+
+ init();
+ EntityMutationResponse response = entityStore.updateByUniqueAttributes(typeRegistry.getEntityTypeByName("B"),
+ Collections.singletonMap(NAME, b1.getAttribute(NAME)),
+ new AtlasEntityWithExtInfo(b1PartialUpdate));
+
+ List<AtlasEntityHeader> updatedEntityHeaders = response.getPartialUpdatedEntities();
+ assertEquals(updatedEntityHeaders.size(), 3);
+
+ AtlasEntitiesWithExtInfo updatedEntities = entityStore.getByIds(ImmutableList.of(a1.getGuid(), a2.getGuid(), b1.getGuid()));
+
+ AtlasEntity b1Entity = updatedEntities.getEntity(b1.getGuid());
+ verifyRelationshipAttributeList(b1Entity, "manyToManyA", ImmutableList.of(getAtlasObjectId(a1), getAtlasObjectId(a2)));
+
+ AtlasEntity a1Entity = updatedEntities.getEntity(a1.getGuid());
+ verifyRelationshipAttributeList(a1Entity, "manyB", ImmutableList.of(getAtlasObjectId(b1)));
+
+ AtlasEntity a2Entity = updatedEntities.getEntity(a2.getGuid());
+ verifyRelationshipAttributeList(a2Entity, "manyB", ImmutableList.of(getAtlasObjectId(b1)));
+ }
+
+ protected abstract void verifyRelationshipAttributeUpdate_NonComposite_OneToOne(AtlasEntity a1, AtlasEntity b);
+
+ protected abstract void verifyRelationshipAttributeUpdate_NonComposite_OneToMany(AtlasEntity entity) throws Exception;
+
+ protected abstract void verifyRelationshipAttributeUpdate_NonComposite_ManyToOne(AtlasEntity a1, AtlasEntity a2, AtlasEntity a3, AtlasEntity b);
+
+ private static void assertObjectIdsContains(List<AtlasObjectId> objectIds, AtlasObjectId objectId) {
+ assertTrue(CollectionUtils.isNotEmpty(objectIds));
+ assertTrue(objectIds.contains(objectId));
+ }
+
+ private static void assertObjectIdEquals(AtlasObjectId objId1, AtlasObjectId objId2) {
+ assertTrue(objId1.equals(objId2));
+ }
+
+ private static List<AtlasObjectId> toAtlasObjectIds(Object objectIds) {
+ if (objectIds instanceof List) {
+ return (List<AtlasObjectId>) objectIds;
+ }
+
+ return null;
+ }
+
+ private static AtlasObjectId toAtlasObjectId(Object objectId) {
+ if (objectId instanceof AtlasObjectId) {
+ return (AtlasObjectId) objectId;
+ }
+
+ return null;
+ }
+
+ private AtlasEntity getEntityFromStore(String guid) throws AtlasBaseException {
+ AtlasEntityWithExtInfo entity = guid != null ? entityStore.getById(guid) : null;
+
+ return entity != null ? entity.getEntity() : null;
+ }
+
+ protected static void verifyRelationshipAttributeList(AtlasEntity entity, String relationshipAttrName, List<AtlasObjectId> expectedValues) {
+ Object refValue = entity.getRelationshipAttribute(relationshipAttrName);
+ assertTrue(refValue instanceof List);
+
+ List<AtlasObjectId> refList = (List<AtlasObjectId>) refValue;
+ assertEquals(refList.size(), expectedValues.size());
+
+ if (expectedValues.size() > 0) {
+ assertTrue(refList.containsAll(expectedValues));
+ }
+ }
+
+ protected static void verifyRelationshipAttributeValue(AtlasEntity entity, String relationshipAttrName, String expectedGuid) {
+ Object refValue = entity.getRelationshipAttribute(relationshipAttrName);
+ if (expectedGuid == null) {
+ assertNull(refValue);
+ }
+ else {
+ assertTrue(refValue instanceof AtlasObjectId);
+ AtlasObjectId referencedObjectId = (AtlasObjectId) refValue;
+ assertEquals(referencedObjectId.getGuid(), expectedGuid);
+ }
}
-}
+}
\ No newline at end of file
[30/39] atlas git commit: ATLAS-1968: updated import documentation
for fileName option usage
Posted by ma...@apache.org.
ATLAS-1968: updated import documentation for fileName option usage
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/3d3be408
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/3d3be408
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/3d3be408
Branch: refs/heads/feature-odf
Commit: 3d3be40848775b43c97f4f94df47591762e78d0b
Parents: 02e4e86
Author: ashutoshm <am...@hortonworks.com>
Authored: Thu Jul 20 09:56:04 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jul 20 10:41:27 2017 -0700
----------------------------------------------------------------------
docs/src/site/twiki/Import-API-Options.twiki | 22 ++++++++++++++++++++
docs/src/site/twiki/Import-API.twiki | 6 +++---
.../atlas/model/impexp/AtlasImportRequest.java | 18 +++++++++++++---
.../atlas/repository/impexp/ImportService.java | 2 +-
4 files changed, 41 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/3d3be408/docs/src/site/twiki/Import-API-Options.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Import-API-Options.twiki b/docs/src/site/twiki/Import-API-Options.twiki
index 8588784..cc09fc7 100644
--- a/docs/src/site/twiki/Import-API-Options.twiki
+++ b/docs/src/site/twiki/Import-API-Options.twiki
@@ -111,3 +111,25 @@ To use the option, set the contents of _importOptions.json_ to:
}
}
</verbatim>
+
+---++++ Specifying File to be Imported From Server Location
+
+In scenario where the file to be imported is present at a location on the server, the _importfile_ API can be used. It behaves like the Import API.
+
+To use the option, set the contents of _importOptions.json_ to:
+<verbatim>
+{
+
+ "options": {
+ "fileName": "/root/fileToBeImported.zip"
+ }
+}
+</verbatim>
+
+_CURL_
+<verbatim>
+curl -g -X POST -u adminuser:password -H "Content-Type: application/json"
+ -H "Cache-Control: no-cache"
+ -d r@importOptions.json
+ "http://localhost:21000/api/atlas/admin/importfile"
+</verbatim>
http://git-wip-us.apache.org/repos/asf/atlas/blob/3d3be408/docs/src/site/twiki/Import-API.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Import-API.twiki b/docs/src/site/twiki/Import-API.twiki
index b5de113..00ac9c4 100644
--- a/docs/src/site/twiki/Import-API.twiki
+++ b/docs/src/site/twiki/Import-API.twiki
@@ -24,7 +24,7 @@ The general approach is:
| _Description_|Provide the path of the file to be imported.|
| _URL_ |_api/atlas/admin/importfile_ |
| _Method_ |_POST_ |
-| _URL Parameters_ |_?FILENAME=<path of file>_ Specify the options as name-value pairs. Use _FILENAME_ to specify the file path. |
+| _URL Parameters_ |_None_ |
| _Data Parameters_|_None_|
| _Success Response_ | _!AtlasImporResult_ is returned as JSON. See details below.|
|_Error Response_|Errors that are handled within the system will be returned as _!AtlasBaseException_. |
@@ -68,8 +68,8 @@ curl -g -X POST -u adminuser:password -H "Content-Type: multipart/form-data"
The call below performs Import of _!QuickStart_ database using a ZIP file available on server.
<verbatim>
-curl -X POST -u adminuser:password -H "Cache-Control: no-cache"
-"http://localhost:21000/api/atlas/admin/importFile?FILENAME=/root/quickStartDB.zip" > quickStartDB-import-result.json
+curl -X POST -u adminuser:password -H "Cache-Control: no-cache" -d ./importOptions.json
+"http://localhost:21000/api/atlas/admin/importFile" > quickStartDB-import-result.json
</verbatim>
Below is the _!AtlasImportResult_ JSON for an import that contains _hive_db_.
http://git-wip-us.apache.org/repos/asf/atlas/blob/3d3be408/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 4d2ac62..304ec5d 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -36,9 +36,11 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@JsonIgnoreProperties(ignoreUnknown=true)
public class AtlasImportRequest implements Serializable {
private static final long serialVersionUID = 1L;
- public static final String TRANSFORMS_KEY = "transforms";
- private static final String START_POSITION_KEY = "startPosition";
- private static final String START_GUID_KEY = "startGuid";
+
+ public static final String TRANSFORMS_KEY = "transforms";
+ private static final String START_POSITION_KEY = "startPosition";
+ private static final String START_GUID_KEY = "startGuid";
+ private static final String FILE_NAME_KEY = "fileName";
private static final String UPDATE_TYPE_DEFINITION_KEY = "updateTypeDefinition";
private Map<String, String> options;
@@ -76,6 +78,16 @@ public class AtlasImportRequest implements Serializable {
}
@JsonIgnore
+ public String getFileName() {
+ return getOptionForKey(FILE_NAME_KEY);
+ }
+
+ @JsonIgnore
+ public void setFileName(String fileName) {
+ setOption(FILE_NAME_KEY, fileName);
+ }
+
+ @JsonIgnore
public String getStartPosition() {
return getOptionForKey(START_POSITION_KEY);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3d3be408/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index 9221717..bdeed7d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -99,7 +99,7 @@ public class ImportService {
public AtlasImportResult run(AtlasImportRequest request, String userName, String hostName, String requestingIP)
throws AtlasBaseException {
- String fileName = (String) request.getOptions().get("FILENAME");
+ String fileName = (String) request.getFileName();
if (StringUtils.isBlank(fileName)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "FILENAME parameter not found");
[29/39] atlas git commit: ATLAS-1959: Enhance relationship attributes
to support cardinality mappings
Posted by ma...@apache.org.
ATLAS-1959: Enhance relationship attributes to support cardinality mappings
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/02e4e86b
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/02e4e86b
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/02e4e86b
Branch: refs/heads/feature-odf
Commit: 02e4e86b5adeb52044ba09a978c8ccbb2200d67d
Parents: f74e43c
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Wed Jul 19 12:05:13 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Jul 19 18:39:14 2017 -0700
----------------------------------------------------------------------
addons/models/0010-base_model.json | 12 +-
addons/models/0030-hive_model.json | 36 +-
addons/models/0050-falcon_model.json | 18 +-
addons/models/0060-hbase_model.json | 16 +-
addons/models/0080-storm_model.json | 6 +-
.../atlas/model/instance/AtlasEntity.java | 14 +-
.../atlas/model/instance/AtlasRelationship.java | 2 -
.../model/typedef/AtlasRelationshipDef.java | 2 +
.../model/typedef/AtlasRelationshipEndDef.java | 25 +-
.../org/apache/atlas/type/AtlasEntityType.java | 12 +
.../atlas/type/AtlasRelationshipType.java | 55 ++-
.../org/apache/atlas/type/AtlasStructType.java | 32 +-
.../apache/atlas/TestRelationshipUtilsV2.java | 267 +++++++++++
.../test/java/org/apache/atlas/TestUtilsV2.java | 1 -
.../atlas/repository/graph/GraphHelper.java | 86 +++-
.../graph/v1/AtlasEntityGraphDiscoveryV1.java | 18 +-
.../store/graph/v1/AtlasGraphUtilsV1.java | 2 +-
.../graph/v1/AtlasRelationshipStoreV1.java | 156 ++++---
.../store/graph/v1/DeleteHandlerV1.java | 33 +-
.../store/graph/v1/EntityGraphMapper.java | 240 ++++++++--
.../store/graph/v1/EntityGraphRetriever.java | 235 +++++-----
.../repository/impexp/ImportServiceTest.java | 1 +
.../impexp/ZipFileResourceTestUtils.java | 3 +
.../AtlasRelationshipStoreHardDeleteV1Test.java | 54 +++
.../AtlasRelationshipStoreSoftDeleteV1Test.java | 55 +++
.../graph/v1/AtlasRelationshipStoreV1Test.java | 437 ++++++++++++++++++-
26 files changed, 1489 insertions(+), 329 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json
index 303f379..20996a2 100644
--- a/addons/models/0010-base_model.json
+++ b/addons/models/0010-base_model.json
@@ -102,15 +102,15 @@
"endDef1": {
"type": "DataSet",
"name": "sourceToProcesses",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SET"
},
"endDef2": {
"type": "Process",
"name": "inputs",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET",
- "legacyLabel": "__Process.inputs"
+ "isLegacyAttribute": true
},
"propagateTags": "NONE"
},
@@ -121,14 +121,14 @@
"endDef1": {
"type": "Process",
"name": "outputs",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET",
- "legacyLabel": "__Process.outputs"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "DataSet",
"name": "sinkFromProcesses",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SET"
},
"propagateTags": "NONE"
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0030-hive_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json
index a795f0f..f47a7b9 100644
--- a/addons/models/0030-hive_model.json
+++ b/addons/models/0030-hive_model.json
@@ -530,15 +530,15 @@
"endDef1": {
"type": "hive_db",
"name": "tables",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET"
},
"endDef2": {
"type": "hive_table",
"name": "db",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hive_table.db"
+ "isLegacyAttribute": true
},
"propagateTags": "ONE_TO_TWO"
},
@@ -549,16 +549,16 @@
"endDef1": {
"type": "hive_table",
"name": "columns",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET",
- "legacyLabel": "__hive_table.columns"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "hive_column",
"name": "table",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hive_column.table"
+ "isLegacyAttribute": true
},
"propagateTags": "ONE_TO_TWO"
},
@@ -569,16 +569,16 @@
"endDef1": {
"type": "hive_table",
"name": "partitionKeys",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET",
- "legacyLabel": "__hive_table.partitionKeys"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "hive_column",
"name": "table",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hive_column.table"
+ "isLegacyAttribute": true
},
"propagateTags": "ONE_TO_TWO"
},
@@ -589,16 +589,16 @@
"endDef1": {
"type": "hive_table",
"name": "sd",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SINGLE",
- "legacyLabel": "__hive_table.sd"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "hive_storagedesc",
"name": "table",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hive_storagedesc.table"
+ "isLegacyAttribute": true
},
"propagateTags": "ONE_TO_TWO"
},
@@ -609,15 +609,15 @@
"endDef1": {
"type": "hive_process",
"name": "columnLineages",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET"
},
"endDef2": {
"type": "hive_column_lineage",
"name": "query",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hive_column_lineage.query"
+ "isLegacyAttribute": true
},
"propagateTags": "NONE"
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0050-falcon_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0050-falcon_model.json b/addons/models/0050-falcon_model.json
index 7755fa8..4fa4604 100644
--- a/addons/models/0050-falcon_model.json
+++ b/addons/models/0050-falcon_model.json
@@ -152,14 +152,14 @@
"endDef1": {
"type": "falcon_feed",
"name": "stored-in",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__falcon_feed.stored-in"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "falcon_cluster",
"name": "feeds",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET"
},
"propagateTags": "NONE"
@@ -171,15 +171,15 @@
"endDef1": {
"type": "falcon_cluster",
"name": "processes",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET"
},
"endDef2": {
"type": "falcon_process",
"name": "runs-on",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__falcon_process.runs-on"
+ "isLegacyAttribute": true
},
"propagateTags": "NONE"
},
@@ -190,15 +190,15 @@
"endDef1": {
"type": "falcon_cluster",
"name": "feedCreations",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET"
},
"endDef2": {
"type": "falcon_feed_creation",
"name": "stored-in",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__falcon_feed_creation.stored-in"
+ "isLegacyAttribute": true
},
"propagateTags": "NONE"
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0060-hbase_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0060-hbase_model.json b/addons/models/0060-hbase_model.json
index 1d264df..3e46e06 100644
--- a/addons/models/0060-hbase_model.json
+++ b/addons/models/0060-hbase_model.json
@@ -105,16 +105,16 @@
"endDef1": {
"type": "hbase_table",
"name": "column_families",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET",
- "legacyLabel": "__hbase_table.column_families"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "hbase_column_family",
"name": "table",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hbase_column_family.table"
+ "isLegacyAttribute": true
},
"propagateTags": "ONE_TO_TWO"
},
@@ -125,16 +125,16 @@
"endDef1": {
"type": "hbase_column_family",
"name": "columns",
- "isContainer": "true",
+ "isContainer": true,
"cardinality": "SET",
- "legacyLabel": "__hbase_column_family.columns"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "hbase_column",
"name": "column_family",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SINGLE",
- "legacyLabel": "__hbase_column.column_family"
+ "isLegacyAttribute": true
},
"propagateTags": "ONE_TO_TWO"
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/addons/models/0080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json
index 25360ff..b008c7a 100644
--- a/addons/models/0080-storm_model.json
+++ b/addons/models/0080-storm_model.json
@@ -151,14 +151,14 @@
"endDef1": {
"type": "storm_topology",
"name": "nodes",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SET",
- "legacyLabel": "__storm_topology.nodes"
+ "isLegacyAttribute": true
},
"endDef2": {
"type": "storm_node",
"name": "topolgies",
- "isContainer": "false",
+ "isContainer": false,
"cardinality": "SET"
},
"propagateTags": "NONE"
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
index 68da6af..365e548 100644
--- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
+++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java
@@ -177,7 +177,7 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
this.relationshipAttributes = relationshipAttributes;
}
- public void addRelationshipAttribute(String name, Object value) {
+ public void setRelationshipAttribute(String name, Object value) {
Map<String, Object> r = this.relationshipAttributes;
if (r != null) {
@@ -190,6 +190,18 @@ public class AtlasEntity extends AtlasStruct implements Serializable {
}
}
+ public Object getRelationshipAttribute(String name) {
+ Map<String, Object> a = this.relationshipAttributes;
+
+ return a != null ? a.get(name) : null;
+ }
+
+ public boolean hasRelationshipAttribute(String name) {
+ Map<String, Object> r = this.relationshipAttributes;
+
+ return r != null ? r.containsKey(name) : false;
+ }
+
public List<AtlasClassification> getClassifications() { return classifications; }
public void setClassifications(List<AtlasClassification> classifications) { this.classifications = classifications; }
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
index 2de9bdf..4188371 100644
--- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
+++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasRelationship.java
@@ -176,8 +176,6 @@ public class AtlasRelationship extends AtlasStruct implements Serializable {
return "-" + Long.toString(s_nextId.getAndIncrement());
}
- public String getRelationshipLabel() { return "r:" + super.getTypeName(); }
-
private void init() {
init(nextInternalId(), null, null, null, null, null, null, null, null, 0L);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
index fc820d4..c17e875 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
@@ -219,6 +219,8 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
return this.endDef2;
}
+ public String getRelationshipLabel() { return "r:" + super.getName(); }
+
public AtlasRelationshipDef(AtlasRelationshipDef other) throws AtlasBaseException {
super(other);
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java
index f80ea89..01e5ce7 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipEndDef.java
@@ -18,7 +18,6 @@
package org.apache.atlas.model.typedef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
-import org.apache.commons.lang.StringUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -61,9 +60,9 @@ public class AtlasRelationshipEndDef implements Serializable {
*/
private Cardinality cardinality;
/**
- * legacy edge label name of the endpoint
+ * When set this indicates that this end is is a legacy attribute
*/
- private String legacyLabel;
+ private boolean isLegacyAttribute;
/**
* Base constructor
@@ -97,15 +96,15 @@ public class AtlasRelationshipEndDef implements Serializable {
* - whether the end is a container or not
*/
public AtlasRelationshipEndDef(String typeName, String name, Cardinality cardinality, boolean isContainer) {
- this(typeName, name, cardinality, isContainer, null);
+ this(typeName, name, cardinality, isContainer, false);
}
- public AtlasRelationshipEndDef(String typeName, String name, Cardinality cardinality, boolean isContainer, String legacyLabel) {
+ public AtlasRelationshipEndDef(String typeName, String name, Cardinality cardinality, boolean isContainer, boolean isLegacyAttribute) {
setType(typeName);
setName(name);
setCardinality(cardinality);
setIsContainer(isContainer);
- setLegacyLabel(legacyLabel);
+ setIsLegacyAttribute(isLegacyAttribute);
}
/**
@@ -118,7 +117,7 @@ public class AtlasRelationshipEndDef implements Serializable {
setName(other.getName());
setIsContainer(other.getIsContainer());
setCardinality(other.getCardinality());
- setLegacyLabel(other.getLegacyLabel());
+ setIsLegacyAttribute(other.isLegacyAttribute);
}
}
@@ -166,11 +165,9 @@ public class AtlasRelationshipEndDef implements Serializable {
return this.cardinality;
}
- public String getLegacyLabel() { return legacyLabel; }
+ public boolean getIsLegacyAttribute() { return isLegacyAttribute; }
- public void setLegacyLabel(String legacyLabel) { this.legacyLabel = legacyLabel; }
-
- public boolean hasLegacyRelation() { return StringUtils.isNotEmpty(getLegacyLabel()) ? true : false; }
+ public void setIsLegacyAttribute(boolean legacyAttribute) { isLegacyAttribute = legacyAttribute; }
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
@@ -182,7 +179,7 @@ public class AtlasRelationshipEndDef implements Serializable {
sb.append(", name==>'").append(name).append('\'');
sb.append(", isContainer==>'").append(isContainer).append('\'');
sb.append(", cardinality==>'").append(cardinality).append('\'');
- sb.append(", legacyLabel==>'").append(legacyLabel).append('\'');
+ sb.append(", isLegacyAttribute==>'").append(isLegacyAttribute).append('\'');
sb.append('}');
return sb;
@@ -200,12 +197,12 @@ public class AtlasRelationshipEndDef implements Serializable {
Objects.equals(name, that.name) &&
isContainer == that.isContainer &&
cardinality == that.cardinality &&
- Objects.equals(legacyLabel, that.legacyLabel);
+ isLegacyAttribute == that.isLegacyAttribute;
}
@Override
public int hashCode() {
- return Objects.hash(type, getName(), isContainer, cardinality, legacyLabel);
+ return Objects.hash(type, getName(), isContainer, cardinality, isLegacyAttribute);
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index e94dd19..e3005ee 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -191,6 +191,8 @@ public class AtlasEntityType extends AtlasStructType {
public Map<String, AtlasAttribute> getRelationshipAttributes() { return relationshipAttributes; }
+ public AtlasAttribute getRelationshipAttribute(String attributeName) { return relationshipAttributes.get(attributeName); }
+
// this method should be called from AtlasRelationshipType.resolveReferencesPhase2()
void addRelationshipAttribute(String attributeName, AtlasAttribute attribute) {
relationshipAttributes.put(attributeName, attribute);
@@ -220,6 +222,16 @@ public class AtlasEntityType extends AtlasStructType {
return relationshipAttributes.containsKey(attributeName);
}
+ public String getQualifiedAttributeName(String attrName) throws AtlasBaseException {
+ if (allAttributes.containsKey(attrName)) {
+ return allAttributes.get(attrName).getQualifiedName();
+ } else if (relationshipAttributes.containsKey(attrName)) {
+ return relationshipAttributes.get(attrName).getQualifiedName();
+ }
+
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entityDef.getName());
+ }
+
@Override
public AtlasEntity createDefaultValue() {
AtlasEntity ret = new AtlasEntity(entityDef.getName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
index 841b66f..934dffc 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
@@ -30,6 +30,10 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN;
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT;
+
/**
* class that implements behaviour of an relationship-type.
*/
@@ -97,19 +101,41 @@ public class AtlasRelationshipType extends AtlasStructType {
// if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
// if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
// if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
- if (!endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
- relationshipLabel = relationshipDef.getName();
-
- } else if (endDef1.hasLegacyRelation() && !endDef2.hasLegacyRelation()) {
- relationshipLabel = endDef1.getLegacyLabel();
-
- } else if (!endDef1.hasLegacyRelation() && endDef2.hasLegacyRelation()) {
- relationshipLabel = endDef2.getLegacyLabel();
+ if (!endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
+ relationshipLabel = relationshipDef.getRelationshipLabel();
+ } else if (endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
+ relationshipLabel = getLegacyEdgeLabel(end1Type, endDef1.getName());
+ } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
+ relationshipLabel = getLegacyEdgeLabel(end2Type, endDef2.getName());
}
addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel);
addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel);
+
+ // add relationship edge direction information
+ addRelationshipEdgeDirection();
+ }
+
+ private void addRelationshipEdgeDirection() {
+ AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
+ AtlasAttribute end1Attribute = end1Type.getRelationshipAttribute(endDef1.getName());
+ AtlasAttribute end2Attribute = end2Type.getRelationshipAttribute(endDef2.getName());
+
+ //default relationship edge direction is end1 (out) -> end2 (in)
+ AtlasRelationshipEdgeDirection end1Direction = OUT;
+ AtlasRelationshipEdgeDirection end2Direction = IN;
+
+ if (endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
+ end2Direction = OUT;
+ } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
+ end1Direction = IN;
+ end2Direction = OUT;
+ }
+
+ end1Attribute.setRelationshipEdgeDirection(end1Direction);
+ end2Attribute.setRelationshipEdgeDirection(end2Direction);
}
@Override
@@ -229,7 +255,7 @@ public class AtlasRelationshipType extends AtlasStructType {
// if relationshipLabel is null, then legacyLabel is mentioned at both ends,
// use the respective end's legacyLabel as relationshipLabel
if (relationshipLabel == null) {
- relationshipLabel = endDef.getLegacyLabel();
+ relationshipLabel = getLegacyEdgeLabel(entityType, attrName);
}
if (attribute == null) { //attr doesn't exist in type - is a new relationship attribute
@@ -251,4 +277,15 @@ public class AtlasRelationshipType extends AtlasStructType {
entityType.addRelationshipAttributeType(attrName, this);
}
+
+ private String getLegacyEdgeLabel(AtlasEntityType entityType, String attributeName) {
+ String ret = null;
+ AtlasAttribute attribute = entityType.getAttribute(attributeName);
+
+ if (attribute != null) {
+ ret = "__" + attribute.getQualifiedName();
+ }
+
+ return ret;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index f97d767..b390a97 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -601,15 +601,16 @@ public class AtlasStructType extends AtlasType {
}
public static class AtlasAttribute {
- private final AtlasStructType definedInType;
- private final AtlasType attributeType;
- private final AtlasAttributeDef attributeDef;
- private final String qualifiedName;
- private final String vertexPropertyName;
- private final boolean isOwnedRef;
- private final String inverseRefAttributeName;
- private AtlasAttribute inverseRefAttribute;
- private String relationshipEdgeLabel;
+ private final AtlasStructType definedInType;
+ private final AtlasType attributeType;
+ private final AtlasAttributeDef attributeDef;
+ private final String qualifiedName;
+ private final String vertexPropertyName;
+ private final boolean isOwnedRef;
+ private final String inverseRefAttributeName;
+ private AtlasAttribute inverseRefAttribute;
+ private String relationshipEdgeLabel;
+ private AtlasRelationshipEdgeDirection relationshipEdgeDirection;
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType, String relationshipLabel) {
this.definedInType = definedInType;
@@ -637,8 +638,9 @@ public class AtlasStructType extends AtlasType {
}
}
- this.isOwnedRef = isOwnedRef;
- this.inverseRefAttributeName = inverseRefAttribute;
+ this.isOwnedRef = isOwnedRef;
+ this.inverseRefAttributeName = inverseRefAttribute;
+ this.relationshipEdgeDirection = AtlasRelationshipEdgeDirection.OUT;
}
public AtlasAttribute(AtlasStructType definedInType, AtlasAttributeDef attrDef, AtlasType attributeType) {
@@ -677,6 +679,12 @@ public class AtlasStructType extends AtlasType {
public void setRelationshipEdgeLabel(String relationshipEdgeLabel) { this.relationshipEdgeLabel = relationshipEdgeLabel; }
+ public AtlasRelationshipEdgeDirection getRelationshipEdgeDirection() { return relationshipEdgeDirection; }
+
+ public void setRelationshipEdgeDirection(AtlasRelationshipEdgeDirection relationshipEdgeDirection) {
+ this.relationshipEdgeDirection = relationshipEdgeDirection;
+ }
+
public static String getEdgeLabel(String property) {
return "__" + property;
}
@@ -721,5 +729,7 @@ public class AtlasStructType extends AtlasType {
new String[] { "$", "_d" },
new String[] { "%", "_p" },
};
+
+ public enum AtlasRelationshipEdgeDirection { IN, OUT }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java
new file mode 100755
index 0000000..98be2b8
--- /dev/null
+++ b/intg/src/test/java/org/apache/atlas/TestRelationshipUtilsV2.java
@@ -0,0 +1,267 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
+import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.commons.lang.StringUtils;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+
+import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.getArrayTypeName;
+import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.getMapTypeName;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.AGGREGATION;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory.ASSOCIATION;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SET;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE;
+import static org.apache.atlas.type.AtlasTypeUtil.createClassTypeDef;
+import static org.apache.atlas.type.AtlasTypeUtil.createOptionalAttrDef;
+import static org.apache.atlas.type.AtlasTypeUtil.createRequiredAttrDef;
+import static org.apache.atlas.type.AtlasTypeUtil.createStructTypeDef;
+import static org.apache.atlas.type.AtlasTypeUtil.createTraitTypeDef;
+import static org.apache.atlas.type.AtlasTypeUtil.createUniqueRequiredAttrDef;
+import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId;
+
+/**
+ * Test utility class for relationship.
+ */
+public final class TestRelationshipUtilsV2 {
+
+ public static final String ORG_LEVEL_TYPE = "OrgLevel";
+ public static final String SECURITY_CLEARANCE_TYPE = "SecurityClearance";
+ public static final String ADDRESS_TYPE = "Address";
+ public static final String PERSON_TYPE = "Person";
+ public static final String MANAGER_TYPE = "Manager";
+ public static final String DEPARTMENT_TYPE = "Department";
+ public static final String EMPLOYEE_TYPE = "Employee";
+ public static final String EMPLOYEE_DEPARTMENT_TYPE = "EmployeeDepartment";
+ public static final String EMPLOYEE_MANAGER_TYPE = "EmployeeManager";
+ public static final String EMPLOYEE_MENTOR_TYPE = "EmployeeMentor";
+ public static final String TYPE_A = "A";
+ public static final String TYPE_B = "B";
+ public static final String DEFAULT_VERSION = "1.0";
+
+
+ private TestRelationshipUtilsV2() { }
+
+ public static AtlasTypesDef getDepartmentEmployeeTypes() throws AtlasBaseException {
+
+ /******* Person Type *******/
+ AtlasEntityDef personType = createClassTypeDef(PERSON_TYPE, description(PERSON_TYPE), superType(null),
+ createUniqueRequiredAttrDef("name", "string"),
+ createOptionalAttrDef("address", ADDRESS_TYPE),
+ createOptionalAttrDef("birthday", "date"),
+ createOptionalAttrDef("hasPets", "boolean"),
+ createOptionalAttrDef("numberOfCars", "byte"),
+ createOptionalAttrDef("houseNumber", "short"),
+ createOptionalAttrDef("carMileage", "int"),
+ createOptionalAttrDef("age", "float"),
+ createOptionalAttrDef("numberOfStarsEstimate", "biginteger"),
+ createOptionalAttrDef("approximationOfPi", "bigdecimal"));
+ /******* Employee Type *******/
+ AtlasEntityDef employeeType = createClassTypeDef(EMPLOYEE_TYPE, description(EMPLOYEE_TYPE), superType(PERSON_TYPE),
+ createOptionalAttrDef("orgLevel", ORG_LEVEL_TYPE),
+ createOptionalAttrDef("shares", "long"),
+ createOptionalAttrDef("salary", "double"));
+ /******* Department Type *******/
+ AtlasEntityDef departmentType = createClassTypeDef(DEPARTMENT_TYPE, description(DEPARTMENT_TYPE), superType(null),
+ createUniqueRequiredAttrDef("name", "string"));
+ /******* Manager Type *******/
+ AtlasEntityDef managerType = createClassTypeDef(MANAGER_TYPE, description(MANAGER_TYPE), superType(EMPLOYEE_TYPE));
+ /******* Address Type *******/
+ AtlasStructDef addressType = createStructTypeDef(ADDRESS_TYPE, description(ADDRESS_TYPE),
+ createRequiredAttrDef("street", "string"),
+ createRequiredAttrDef("city", "string"));
+ /******* Organization Level Type *******/
+ AtlasEnumDef orgLevelType = new AtlasEnumDef(ORG_LEVEL_TYPE, description(ORG_LEVEL_TYPE), DEFAULT_VERSION,
+ getOrgLevelElements());
+
+ /******* Security Clearance Type *******/
+ AtlasClassificationDef securityClearanceType = createTraitTypeDef(SECURITY_CLEARANCE_TYPE, description(SECURITY_CLEARANCE_TYPE),
+ superType(null), createRequiredAttrDef("level", "int"));
+
+ /******* [Department -> Employee] Relationship *******/
+ AtlasRelationshipDef employeeDepartmentType = new AtlasRelationshipDef(EMPLOYEE_DEPARTMENT_TYPE, description(EMPLOYEE_DEPARTMENT_TYPE),
+ DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "department", SINGLE),
+ new AtlasRelationshipEndDef(DEPARTMENT_TYPE, "employees", SET, true));
+ /******* [Manager -> Employee] Relationship *******/
+ AtlasRelationshipDef employeeManagerType = new AtlasRelationshipDef(EMPLOYEE_MANAGER_TYPE, description(EMPLOYEE_MANAGER_TYPE),
+ DEFAULT_VERSION, AGGREGATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "manager", SINGLE),
+ new AtlasRelationshipEndDef(MANAGER_TYPE, "subordinates", SET, true));
+
+ /******* [Mentor -> Employee] Relationship *******/
+ AtlasRelationshipDef employeeMentorType = new AtlasRelationshipDef(EMPLOYEE_MENTOR_TYPE, description(EMPLOYEE_MENTOR_TYPE),
+ DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentor", SINGLE),
+ new AtlasRelationshipEndDef(EMPLOYEE_TYPE, "mentees", SET));
+
+ return new AtlasTypesDef(ImmutableList.of(orgLevelType),
+ ImmutableList.of(addressType),
+ ImmutableList.of(securityClearanceType),
+ ImmutableList.of(personType, employeeType, departmentType, managerType),
+ ImmutableList.of(employeeDepartmentType, employeeManagerType, employeeMentorType));
+ }
+
+ public static AtlasEntitiesWithExtInfo getDepartmentEmployeeInstances() {
+ AtlasEntitiesWithExtInfo ret = new AtlasEntitiesWithExtInfo();
+
+ /******* Department - HR *******/
+ AtlasEntity hrDept = new AtlasEntity(DEPARTMENT_TYPE, "name", "hr");
+
+ /******* Address *******/
+ AtlasStruct janeAddr = new AtlasStruct(ADDRESS_TYPE);
+ janeAddr.setAttribute("street", "Great America Parkway");
+ janeAddr.setAttribute("city", "Santa Clara");
+
+ AtlasStruct juliusAddr = new AtlasStruct(ADDRESS_TYPE);
+ juliusAddr.setAttribute("street", "Madison Ave");
+ juliusAddr.setAttribute("city", "Newtonville");
+
+ AtlasStruct maxAddr = new AtlasStruct(ADDRESS_TYPE);
+ maxAddr.setAttribute("street", "Ripley St");
+ maxAddr.setAttribute("city", "Newton");
+
+ AtlasStruct johnAddr = new AtlasStruct(ADDRESS_TYPE);
+ johnAddr.setAttribute("street", "Stewart Drive");
+ johnAddr.setAttribute("city", "Sunnyvale");
+
+ /******* Manager - Jane (John and Max subordinates) *******/
+ AtlasEntity jane = new AtlasEntity(MANAGER_TYPE);
+ jane.setAttribute("name", "Jane");
+ jane.setRelationshipAttribute("department", getAtlasObjectId(hrDept));
+ jane.setAttribute("address", janeAddr);
+
+ /******* Manager - Julius (no subordinates) *******/
+ AtlasEntity julius = new AtlasEntity(MANAGER_TYPE);
+ julius.setAttribute("name", "Julius");
+ julius.setRelationshipAttribute("department", getAtlasObjectId(hrDept));
+ julius.setAttribute("address", juliusAddr);
+
+ /******* Employee - Max (Manager: Jane, Mentor: Julius) *******/
+ AtlasEntity max = new AtlasEntity(EMPLOYEE_TYPE);
+ max.setAttribute("name", "Max");
+ max.setRelationshipAttribute("department", getAtlasObjectId(hrDept));
+ max.setAttribute("address", maxAddr);
+ max.setRelationshipAttribute("manager", getAtlasObjectId(jane));
+ max.setRelationshipAttribute("mentor", getAtlasObjectId(julius));
+ max.setAttribute("birthday",new Date(1979, 3, 15));
+ max.setAttribute("hasPets", true);
+ max.setAttribute("age", 36);
+ max.setAttribute("numberOfCars", 2);
+ max.setAttribute("houseNumber", 17);
+ max.setAttribute("carMileage", 13);
+ max.setAttribute("shares", Long.MAX_VALUE);
+ max.setAttribute("salary", Double.MAX_VALUE);
+ max.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000000000000"));
+ max.setAttribute("approximationOfPi", new BigDecimal("3.1415926535897932"));
+
+ /******* Employee - John (Manager: Jane, Mentor: Max) *******/
+ AtlasEntity john = new AtlasEntity(EMPLOYEE_TYPE);
+ john.setAttribute("name", "John");
+ john.setRelationshipAttribute("department", getAtlasObjectId(hrDept));
+ john.setAttribute("address", johnAddr);
+ john.setRelationshipAttribute("manager", getAtlasObjectId(jane));
+ john.setRelationshipAttribute("mentor", getAtlasObjectId(max));
+ john.setAttribute("birthday",new Date(1950, 5, 15));
+ john.setAttribute("hasPets", true);
+ john.setAttribute("numberOfCars", 1);
+ john.setAttribute("houseNumber", 153);
+ john.setAttribute("carMileage", 13364);
+ john.setAttribute("shares", 15000);
+ john.setAttribute("salary", 123345.678);
+ john.setAttribute("age", 50);
+ john.setAttribute("numberOfStarsEstimate", new BigInteger("1000000000000000000000"));
+ john.setAttribute("approximationOfPi", new BigDecimal("3.141592653589793238462643383279502884197169399375105820974944592307816406286"));
+
+ ret.addEntity(hrDept);
+ ret.addEntity(jane);
+ ret.addEntity(julius);
+ ret.addEntity(max);
+ ret.addEntity(john);
+
+ return ret;
+ }
+
+ public static AtlasTypesDef getInverseReferenceTestTypes() throws AtlasBaseException {
+ AtlasEntityDef aType = createClassTypeDef(TYPE_A, superType(null), createUniqueRequiredAttrDef("name", "string"));
+ AtlasEntityDef bType = createClassTypeDef(TYPE_B, superType(null), createUniqueRequiredAttrDef("name", "string"));
+
+ AtlasRelationshipDef relationshipType1 = new AtlasRelationshipDef("TypeA_to_TypeB_on_b", description("TypeA_to_TypeB_on_b"),
+ DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(TYPE_A, "b", SINGLE),
+ new AtlasRelationshipEndDef(TYPE_B, "a", SINGLE));
+
+ AtlasRelationshipDef relationshipType2 = new AtlasRelationshipDef("TypeA_to_TypeB_on_oneB", description("TypeA_to_TypeB_on_oneB"),
+ DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(TYPE_A, "oneB", SINGLE),
+ new AtlasRelationshipEndDef(TYPE_B, "manyA", SET));
+
+ AtlasRelationshipDef relationshipType3 = new AtlasRelationshipDef("TypeA_to_TypeB_on_manyB", description("TypeA_to_TypeB_on_manyB"),
+ DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(TYPE_A, "manyB", SET),
+ new AtlasRelationshipEndDef(TYPE_B, "manyToManyA", SET));
+
+ AtlasRelationshipDef relationshipType4 = new AtlasRelationshipDef("TypeB_to_TypeA_on_mappedFromA", description("TypeB_to_TypeA_on_mappedFromA"),
+ DEFAULT_VERSION, ASSOCIATION, ONE_TO_TWO,
+ new AtlasRelationshipEndDef(TYPE_B, "mappedFromA", SINGLE),
+ new AtlasRelationshipEndDef(TYPE_A, "mapToB", SET));
+
+ return new AtlasTypesDef(ImmutableList.<AtlasEnumDef>of(), ImmutableList.<AtlasStructDef>of(),
+ ImmutableList.<AtlasClassificationDef>of(), ImmutableList.of(aType, bType),
+ ImmutableList.of(relationshipType1, relationshipType2, relationshipType3, relationshipType4));
+ }
+
+ private static List<AtlasEnumElementDef> getOrgLevelElements() {
+ return Arrays.asList(
+ new AtlasEnumElementDef("L1", description("L1"), 1),
+ new AtlasEnumElementDef("L2", description("L2"), 2),
+ new AtlasEnumElementDef("L3", description("L3"), 3)
+ );
+ }
+
+ private static String description(String typeName) {
+ return typeName + " description";
+ }
+
+ private static ImmutableSet<String> superType(String superTypeName) {
+ return StringUtils.isNotEmpty(superTypeName) ? ImmutableSet.of(superTypeName) : ImmutableSet.<String>of();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 9774583..14614f1 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -35,7 +35,6 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
-import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.lang.RandomStringUtils;
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 6f6d74b..c47a89e 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -26,8 +26,10 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasEntity.Status;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -39,6 +41,7 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
@@ -344,12 +347,46 @@ public final class GraphHelper {
return null;
}
+ public Iterator<AtlasEdge> getIncomingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
+ return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.IN, edgeLabel);
+ }
+
public Iterator<AtlasEdge> getOutGoingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
}
- public Iterator<AtlasEdge> getBothEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
- return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.BOTH, edgeLabel);
+ public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) {
+ AtlasEdge ret;
+
+ switch (edgeDirection) {
+ case IN:
+ ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.IN);
+ break;
+
+ case OUT:
+ default:
+ ret = getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.OUT);
+ break;
+ }
+
+ return ret;
+ }
+
+ public Iterator<AtlasEdge> getEdgesForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) {
+ Iterator<AtlasEdge> ret;
+
+ switch (edgeDirection) {
+ case IN:
+ ret = getIncomingEdgesByLabel(vertex, edgeLabel);
+ break;
+
+ case OUT:
+ default:
+ ret = getOutGoingEdgesByLabel(vertex, edgeLabel);
+ break;
+ }
+
+ return ret;
}
/**
@@ -360,7 +397,11 @@ public final class GraphHelper {
* @return
*/
public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel) {
- Iterator<AtlasEdge> iterator = getAdjacentEdgesByLabel(vertex, AtlasEdgeDirection.OUT, edgeLabel);
+ return getEdgeForLabel(vertex, edgeLabel, AtlasEdgeDirection.OUT);
+ }
+
+ public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel, AtlasEdgeDirection edgeDirection) {
+ Iterator<AtlasEdge> iterator = getAdjacentEdgesByLabel(vertex, edgeDirection, edgeLabel);
AtlasEdge latestDeletedEdge = null;
long latestDeletedEdgeTime = Long.MIN_VALUE;
@@ -1280,4 +1321,43 @@ public final class GraphHelper {
return ret;
}
+
+ public static boolean isRelationshipEdge(AtlasEdge edge) {
+ if (edge == null) {
+ return false;
+ }
+
+ String edgeLabel = edge.getLabel();
+
+ return StringUtils.isNotEmpty(edge.getLabel()) ? edgeLabel.startsWith("r:") : false;
+ }
+
+ public static AtlasObjectId getReferenceObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection) {
+ AtlasObjectId ret = null;
+
+ if (relationshipDirection == AtlasRelationshipEdgeDirection.OUT) {
+ ret = new AtlasObjectId(getGuid(edge.getInVertex()), getTypeName(edge.getInVertex()));
+
+ } else if (relationshipDirection == AtlasRelationshipEdgeDirection.IN) {
+ ret = new AtlasObjectId(getGuid(edge.getOutVertex()), getTypeName(edge.getOutVertex()));
+ }
+
+ return ret;
+ }
+
+ public static AtlasObjectId getCurrentObjectId(AtlasEdge edge, AtlasRelationshipEdgeDirection relationshipDirection) {
+ String typeName = null;
+ String guid = null;
+
+ if (relationshipDirection == AtlasRelationshipEdgeDirection.OUT) {
+ typeName = GraphHelper.getTypeName(edge.getOutVertex());
+ guid = GraphHelper.getGuid(edge.getOutVertex());
+
+ } else if (relationshipDirection == AtlasRelationshipEdgeDirection.IN) {
+ typeName = GraphHelper.getTypeName(edge.getInVertex());
+ guid = GraphHelper.getGuid(edge.getInVertex());
+ }
+
+ return new AtlasObjectId(guid, typeName);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
index 12e8bb1..0210a11 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
@@ -286,7 +286,7 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
return;
}
- AtlasStruct struct;
+ final AtlasStruct struct;
if (val instanceof AtlasStruct) {
struct = (AtlasStruct) val;
@@ -298,6 +298,10 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
throw new AtlasBaseException(AtlasErrorCode.INVALID_STRUCT_VALUE, val.toString());
}
+ visitStruct(structType, struct);
+ }
+
+ void visitStruct(AtlasStructType structType, AtlasStruct struct) throws AtlasBaseException {
for (AtlasAttribute attribute : structType.getAllAttributes().values()) {
AtlasType attrType = attribute.getAttributeType();
Object attrVal = struct.getAttribute(attribute.getName());
@@ -306,6 +310,16 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
}
}
+ void visitEntity(AtlasEntityType entityType, AtlasEntity entity) throws AtlasBaseException {
+ visitStruct(entityType, entity);
+
+ for (AtlasAttribute attribute : entityType.getRelationshipAttributes().values()) {
+ AtlasType attrType = attribute.getAttributeType();
+ Object attrVal = entity.getRelationshipAttribute(attribute.getName());
+
+ visitAttribute(attrType, attrVal);
+ }
+ }
void walkEntityGraph(AtlasEntity entity) throws AtlasBaseException {
if (entity == null) {
@@ -316,7 +330,7 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
recordObjectReference(entity.getGuid());
- visitStruct(type, entity);
+ visitEntity(type, entity);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
index 43f2c55..948d9dd 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasGraphUtilsV1.java
@@ -94,8 +94,8 @@ public class AtlasGraphUtilsV1 {
public static String getQualifiedAttributePropertyKey(AtlasStructType fromType, String attributeName) throws AtlasBaseException {
switch (fromType.getTypeCategory()) {
- case STRUCT:
case ENTITY:
+ case STRUCT:
case CLASSIFICATION:
return fromType.getQualifiedAttributeName(attributeName);
default:
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
index 3ff6fbe..49e08a0 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
@@ -29,9 +29,11 @@ import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -78,35 +80,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
- AtlasRelationship ret;
-
- // create relationship between two vertex
- try {
- AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship);
-
- if (relationshipEdge == null) {
- relationshipEdge = createRelationshipEdge(end1Vertex, end2Vertex, relationship);
-
- AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
-
- if (MapUtils.isNotEmpty(relationType.getAllAttributes())) {
- for (AtlasAttribute attr : relationType.getAllAttributes().values()) {
- String attrName = attr.getName();
- Object attrValue = relationship.getAttribute(attrName);
-
- AtlasGraphUtilsV1.setProperty(relationshipEdge, attr.getVertexPropertyName(), attrValue);
- }
- }
-
- ret = mapEdgeToAtlasRelationship(relationshipEdge);
-
- } else {
- throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS, relationship.getTypeName(),
- relationship.getEnd1().getGuid(), relationship.getEnd2().getGuid());
- }
- } catch (RepositoryException e) {
- throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
- }
+ AtlasRelationship ret = createRelationship(relationship, end1Vertex, end2Vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== create({}): {}", relationship, ret);
@@ -117,31 +91,6 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
@Override
@GraphTransaction
- public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> getOrCreate({})", relationship);
- }
-
- validateRelationship(relationship);
-
- AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
- AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
- AtlasRelationship ret;
-
- // check if relationship exists
- AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship);
-
- ret = (relationshipEdge != null) ? mapEdgeToAtlasRelationship(relationshipEdge) : create(relationship);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== getOrCreate({}): {}", relationship, ret);
- }
-
- return ret;
- }
-
- @Override
- @GraphTransaction
public AtlasRelationship update(AtlasRelationship relationship) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> update({})", relationship);
@@ -196,6 +145,70 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
}
}
+ public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getOrCreate({})", relationship);
+ }
+
+ validateRelationship(relationship);
+
+ AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
+ AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
+ AtlasRelationship ret;
+
+ // check if relationship exists
+ AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship);
+
+ if (relationshipEdge != null) {
+ ret = mapEdgeToAtlasRelationship(relationshipEdge);
+
+ } else {
+ validateRelationship(relationship);
+ ret = createRelationship(relationship, end1Vertex, end2Vertex);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getOrCreate({}): {}", relationship, ret);
+ }
+
+ return ret;
+ }
+
+ private AtlasRelationship createRelationship(AtlasRelationship relationship, AtlasVertex end1Vertex, AtlasVertex end2Vertex)
+ throws AtlasBaseException {
+ AtlasRelationship ret;
+
+ try {
+ AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship);
+
+ if (relationshipEdge == null) {
+ relationshipEdge = createRelationshipEdge(end1Vertex, end2Vertex, relationship);
+
+ AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
+
+ if (MapUtils.isNotEmpty(relationType.getAllAttributes())) {
+ for (AtlasAttribute attr : relationType.getAllAttributes().values()) {
+ String attrName = attr.getName();
+ String attrVertexProperty = attr.getVertexPropertyName();
+ Object attrValue = relationship.getAttribute(attrName);
+
+ AtlasGraphUtilsV1.setProperty(relationshipEdge, attrVertexProperty, attrValue);
+ }
+ }
+
+ ret = mapEdgeToAtlasRelationship(relationshipEdge);
+
+ } else {
+ throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS, relationship.getTypeName(),
+ relationship.getEnd1().getGuid(), relationship.getEnd2().getGuid());
+ }
+ } catch (RepositoryException e) {
+ throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e);
+ }
+
+ return ret;
+ }
+
private void validateRelationship(AtlasRelationship relationship) throws AtlasBaseException {
if (relationship == null) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "AtlasRelationship is null");
@@ -229,6 +242,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
}
validateEnd(relationship.getEnd1());
+
validateEnd(relationship.getEnd2());
validateAndNormalize(relationship);
@@ -273,7 +287,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
type.getNormalizedValue(relationship);
}
- private AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
+ public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship);
AtlasEdge ret = graphHelper.getEdgeForLabel(fromVertex, relationshipLabel);
@@ -331,31 +345,29 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
}
private String getRelationshipEdgeLabel(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
-
- String ret = relationship.getRelationshipLabel();
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
+ String ret = relationshipType.getRelationshipDef().getRelationshipLabel();
AtlasRelationshipEndDef endDef1 = relationshipType.getRelationshipDef().getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipType.getRelationshipDef().getEndDef2();
Set<String> fromVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV1.getTypeName(fromVertex));
Set<String> toVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV1.getTypeName(toVertex));
+ AtlasAttribute attribute = null;
// validate entity type and all its supertypes contains relationshipDefs end type
- // e.g. [ hive_process -> hive_table] -> [ Process -> DataSet ]
+ // e.g. [hive_process -> hive_table] -> [Process -> DataSet]
if (fromVertexTypes.contains(endDef1.getType()) && toVertexTypes.contains(endDef2.getType())) {
- String attributeName = endDef1.getName();
- AtlasAttribute endAttribute = relationshipType.getEnd1Type().getAttribute(attributeName);
+ String attributeName = endDef1.getName();
- if (endAttribute != null) {
- ret = endAttribute.getRelationshipEdgeLabel();
- }
+ attribute = relationshipType.getEnd1Type().getRelationshipAttribute(attributeName);
} else if (fromVertexTypes.contains(endDef2.getType()) && toVertexTypes.contains(endDef1.getType())) {
- String attributeName = endDef2.getName();
- AtlasAttribute endAttribute = relationshipType.getEnd2Type().getAttribute(attributeName);
+ String attributeName = endDef2.getName();
- if (endAttribute != null) {
- ret = endAttribute.getRelationshipEdgeLabel();
- }
+ attribute = relationshipType.getEnd2Type().getRelationshipAttribute(attributeName);
+ }
+
+ if (attribute != null) {
+ ret = attribute.getRelationshipEdgeLabel();
}
return ret;
@@ -391,7 +403,13 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
relationship.setCreateTime(new Date(GraphHelper.getCreatedTime(edge)));
relationship.setUpdateTime(new Date(GraphHelper.getModifiedTime(edge)));
- relationship.setVersion(GraphHelper.getVersion(edge).longValue());
+ Integer version = GraphHelper.getVersion(edge);
+
+ if (version == null) {
+ version = Integer.valueOf(1);
+ }
+
+ relationship.setVersion(version.longValue());
relationship.setStatus(GraphHelper.getEdgeStatus(edge));
AtlasVertex end1Vertex = edge.getOutVertex();
http://git-wip-us.apache.org/repos/asf/atlas/blob/02e4e86b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index d4fdc25..4271376 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -37,6 +37,7 @@ import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.slf4j.Logger;
@@ -53,6 +54,8 @@ import java.util.Set;
import java.util.Stack;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
+import static org.apache.atlas.repository.graph.GraphHelper.getReferenceObjectId;
+import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
import static org.apache.atlas.repository.graph.GraphHelper.string;
public abstract class DeleteHandlerV1 {
@@ -206,7 +209,6 @@ public abstract class DeleteHandlerV1 {
return result;
}
-
/**
* Force delete is used to remove struct/trait in case of entity updates
* @param edge
@@ -217,13 +219,20 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasException
*/
public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned,
- boolean forceDeleteStructTrait) throws AtlasBaseException {
+ boolean forceDeleteStructTrait) throws AtlasBaseException {
+
+ // default edge direction is outward
+ return deleteEdgeReference(edge, typeCategory, isOwned, forceDeleteStructTrait, AtlasRelationshipEdgeDirection.OUT);
+ }
+
+ public boolean deleteEdgeReference(AtlasEdge edge, TypeCategory typeCategory, boolean isOwned, boolean forceDeleteStructTrait,
+ AtlasRelationshipEdgeDirection relationshipDirection) throws AtlasBaseException {
LOG.debug("Deleting {}", string(edge));
boolean forceDelete =
- (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
+ (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION) && forceDeleteStructTrait;
if (typeCategory == TypeCategory.STRUCT || typeCategory == TypeCategory.CLASSIFICATION
- || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
+ || (typeCategory == TypeCategory.OBJECT_ID_TYPE && isOwned)) {
//If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
//If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
//through this delete, hence delete the edge and the reference vertex.
@@ -236,9 +245,19 @@ public abstract class DeleteHandlerV1 {
//If the vertex is of type class, and its not a composite attributes, the reference AtlasVertex' lifecycle is not controlled
//through this delete. Hence just remove the reference edge. Leave the reference AtlasVertex as is
- //If deleting just the edge, reverse attribute should be updated for any references
- //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated
- deleteEdge(edge, true, false);
+ // for relationship edges, inverse vertex's relationship attribute doesn't need to be updated.
+ // only delete the reference relationship edge
+ if (isRelationshipEdge(edge)) {
+ deleteEdge(edge, false);
+
+ AtlasObjectId deletedReferenceObjectId = getReferenceObjectId(edge, relationshipDirection);
+ RequestContextV1.get().recordEntityUpdate(deletedReferenceObjectId);
+ } else {
+ //legacy case - not a relationship edge
+ //If deleting just the edge, reverse attribute should be updated for any references
+ //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated
+ deleteEdge(edge, true, false);
+ }
}
return !softDelete || forceDelete;
}
[34/39] atlas git commit: ATLAS-1967: search UI - render attribute
filter based on browser URL
Posted by ma...@apache.org.
ATLAS-1967: search UI - render attribute filter based on browser URL
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f511f272
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f511f272
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f511f272
Branch: refs/heads/feature-odf
Commit: f511f27267adbd5c602e9c2089aba3cac3a314fa
Parents: 62d85a4
Author: kevalbhatt <kb...@apache.org>
Authored: Fri Jul 21 18:21:14 2017 +0530
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Jul 21 07:18:21 2017 -0700
----------------------------------------------------------------------
dashboardv2/public/js/modules/Modal.js | 11 +-
dashboardv2/public/js/router/Router.js | 47 +---
.../public/js/templates/common/modal.html | 7 +-
.../templates/search/SearchLayoutView_tmpl.html | 4 +-
.../search/SearchResultLayoutView_tmpl.html | 15 --
.../public/js/utils/CommonViewFunction.js | 58 +++++
dashboardv2/public/js/utils/TableLayout.js | 6 +-
.../views/business_catalog/SideNavLayoutView.js | 4 +-
.../public/js/views/search/QueryBuilderView.js | 35 +--
.../js/views/search/SearchDetailLayoutView.js | 3 +-
.../public/js/views/search/SearchLayoutView.js | 214 ++++++++-----------
.../public/js/views/search/SearchQueryView.js | 21 +-
.../js/views/search/SearchResultLayoutView.js | 104 ++++-----
.../public/js/views/tag/TagLayoutView.js | 11 +-
14 files changed, 240 insertions(+), 300 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/modules/Modal.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/modules/Modal.js b/dashboardv2/public/js/modules/Modal.js
index 8b27788..48f385e 100644
--- a/dashboardv2/public/js/modules/Modal.js
+++ b/dashboardv2/public/js/modules/Modal.js
@@ -29,7 +29,7 @@ define(['require', 'backbone', 'hbs!tmpl/common/modal'], function(require, Backb
this.trigger('closeModal');
if (this.options.content && this.options.content.trigger) {
- this.options.content.trigger('closeModal', this);
+ this.options.content.trigger('closeModal', this, event);
}
},
'click .cancel': function(event) {
@@ -38,7 +38,7 @@ define(['require', 'backbone', 'hbs!tmpl/common/modal'], function(require, Backb
this.trigger('closeModal');
if (this.options.content && this.options.content.trigger) {
- this.options.content.trigger('closeModal', this);
+ this.options.content.trigger('closeModal', this, event);
}
},
'click .ok': function(event) {
@@ -47,7 +47,7 @@ define(['require', 'backbone', 'hbs!tmpl/common/modal'], function(require, Backb
this.trigger('ok');
if (this.options.content && this.options.content.trigger) {
- this.options.content.trigger('ok', this);
+ this.options.content.trigger('ok', this, event);
}
if (this.options.okCloses) {
@@ -85,7 +85,8 @@ define(['require', 'backbone', 'hbs!tmpl/common/modal'], function(require, Backb
animate: true,
contentWithFooter: false,
template: template,
- width: null
+ width: null,
+ buttons: null
}, options);
},
@@ -250,4 +251,4 @@ define(['require', 'backbone', 'hbs!tmpl/common/modal'], function(require, Backb
});
return Modal;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/router/Router.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/router/Router.js b/dashboardv2/public/js/router/Router.js
index f705843..f858ee2 100644
--- a/dashboardv2/public/js/router/Router.js
+++ b/dashboardv2/public/js/router/Router.js
@@ -53,10 +53,6 @@ define([
'enumDefCollection': this.enumDefCollection,
'classificationDefCollection': this.classificationDefCollection
}
- this.filterObj = {
- 'tagFilters': JSON.parse(Utils.localStorage.getValue('tagFilters')),
- 'entityFilters': JSON.parse(Utils.localStorage.getValue('entityFilters'))
- }
},
bindCommonEvents: function() {
var that = this;
@@ -128,8 +124,7 @@ define([
if (!App.rSideNav.currentView) {
App.rSideNav.show(new SideNavLayoutView(
_.extend({
- 'url': url,
- 'filterObj': that.filterObj
+ 'url': url
}, that.preFetchedCollectionLists)
));
} else {
@@ -161,7 +156,7 @@ define([
App.rNHeader.show(new Header());
if (!App.rSideNav.currentView) {
App.rSideNav.show(new SideNavLayoutView(
- _.extend({ 'filterObj': that.filterObj }, that.preFetchedCollectionLists)
+ _.extend({}, that.preFetchedCollectionLists)
));
} else {
App.rSideNav.currentView.selectTab();
@@ -198,8 +193,7 @@ define([
}
App.rSideNav.show(new SideNavLayoutView(
_.extend({
- 'tag': tagName,
- 'filterObj': that.filterObj
+ 'tag': tagName
}, that.preFetchedCollectionLists)
));
} else {
@@ -242,8 +236,7 @@ define([
if (!App.rSideNav.currentView) {
App.rSideNav.show(new SideNavLayoutView(
_.extend({
- 'searchVent': that.searchVent,
- 'filterObj': that.filterObj
+ 'searchVent': that.searchVent
}, that.preFetchedCollectionLists)
));
} else {
@@ -259,7 +252,6 @@ define([
_.extend({
'value': paramObj,
'initialView': true,
- 'filterObj': that.filterObj,
'searchVent': that.searchVent
}, that.preFetchedCollectionLists)
));
@@ -277,39 +269,13 @@ define([
'views/business_catalog/SideNavLayoutView',
'views/search/SearchDetailLayoutView'
], function(Header, BusinessCatalogLayoutView, SideNavLayoutView, SearchDetailLayoutView) {
- var paramObj = Utils.getUrlState.getQueryParams(),
- filterObj = that.filterObj
- if (paramObj && paramObj.searchType === "basic") {
- if (paramObj.type) {
- if (_.has(filterObj.entityFilters, paramObj.type)) {
- _.extend(paramObj, {
- 'entityFilters': +new Date()
- })
- }
- }
- if (paramObj.tag) {
- if (_.has(filterObj.entityFilters, paramObj.type)) {
- _.extend(paramObj, {
- 'tagFilters': +new Date()
- })
- }
- }
- Utils.setUrl({
- url: '#!/search/searchResult',
- trigger: false,
- urlParams: paramObj,
- updateTabState: function() {
- return { searchUrl: this.url, stateChanged: true };
- },
- });
- }
+ var paramObj = Utils.getUrlState.getQueryParams();
App.rNHeader.show(new Header());
if (!App.rSideNav.currentView) {
App.rSideNav.show(new SideNavLayoutView(
_.extend({
'value': paramObj,
- 'searchVent': that.searchVent,
- 'filterObj': that.filterObj
+ 'searchVent': that.searchVent
}, that.preFetchedCollectionLists)
));
} else {
@@ -320,7 +286,6 @@ define([
_.extend({
'value': paramObj,
'searchVent': that.searchVent,
- 'filterObj': that.filterObj,
'initialView': (paramObj.type || (paramObj.dslChecked == "true" ? "" : paramObj.tag) || (paramObj.query ? paramObj.query.trim() : "")).length === 0
}, that.preFetchedCollectionLists)
));
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/templates/common/modal.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/templates/common/modal.html b/dashboardv2/public/js/templates/common/modal.html
index 6f28d0a..ef59290 100644
--- a/dashboardv2/public/js/templates/common/modal.html
+++ b/dashboardv2/public/js/templates/common/modal.html
@@ -31,11 +31,14 @@
<div class="modal-body">{{content}}</div>
{{#if showFooter}}
<div class="modal-footer">
- {{#if allowCancel}} {{#if cancelText}}
+ {{#if buttons}} {{#each buttons}}
+ <button type="button" class="btn btn-atlas {{this.btnClass}}">{{this.text}}</button>
+ {{/each}} {{else}} {{#if allowCancel}} {{#if cancelText}}
<button type="button" class="btn btn-atlas cancel">{{tt cancelText}}</button>
{{/if}} {{/if}}
<button type="button" class="btn btn-atlas ok">{{tt okText}}</button>
+ {{/if}}
</div>
{{/if}} {{/if}}
</div>
-</div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/templates/search/SearchLayoutView_tmpl.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/templates/search/SearchLayoutView_tmpl.html b/dashboardv2/public/js/templates/search/SearchLayoutView_tmpl.html
index a23057f..911522e 100644
--- a/dashboardv2/public/js/templates/search/SearchLayoutView_tmpl.html
+++ b/dashboardv2/public/js/templates/search/SearchLayoutView_tmpl.html
@@ -40,7 +40,7 @@
<select data-id="typeLOV"></select>
</div>
<div class="col-sm-2 no-padding temFilterBtn">
- <button type="button" class="btn btn-atlasAction btn-atlas pull-right typeLOV" title="Entity Attribute Filter" data-id="typeAttrFilter"><i class="fa fa-filter"></i></button>
+ <button type="button" class="btn btn-atlasAction btn-atlas pull-right typeLOV active" title="Entity Attribute Filter" data-id="typeAttrFilter"><i class="fa fa-filter"></i></button>
</div>
</div>
</div>
@@ -51,7 +51,7 @@
<select data-id="tagLOV"></select>
</div>
<div class="col-sm-2 no-padding">
- <button type="button" class="btn btn-atlasAction btn-atlas pull-right" title="Tag Attribute Filter" data-id="tagAttrFilter"><i class="fa fa-filter"></i></button>
+ <button type="button" class="btn btn-atlasAction btn-atlas pull-right active" title="Tag Attribute Filter" data-id="tagAttrFilter"><i class="fa fa-filter"></i></button>
</div>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/templates/search/SearchResultLayoutView_tmpl.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/templates/search/SearchResultLayoutView_tmpl.html b/dashboardv2/public/js/templates/search/SearchResultLayoutView_tmpl.html
index fc2fe8c..91f1a67 100644
--- a/dashboardv2/public/js/templates/search/SearchResultLayoutView_tmpl.html
+++ b/dashboardv2/public/js/templates/search/SearchResultLayoutView_tmpl.html
@@ -20,21 +20,6 @@
<div class="fontLoader">
<i class="fa fa-refresh fa-spin-custom"></i>
</div>
- <div>
- <div id="r_searchQuery">
- <div class="panel panel-default" id="filterPanel">
- <div class="panel-heading clearfix">
- <h4 class="panel-title pull-left">Type,Tag filter/s</h4>
- <div class="btn-group pull-right">
- <button type="button" id="expand_collapse_panel" class="expand_collapse_panel" title="Collapse"><i class="fa fa-chevron-up" aria-hidden="true"></i></button>
- </div>
- </div>
- <div id="panel_body" class="panel-body collapse in" align="center">
- <p class="filterQuery" id="filterQuery"></p>
- </div>
- </div>
- </div>
- </div>
<div class="ellipsis" style="display: none;"><span class="searchResult" style=" font-size: 16px;"></span>
<a href="javascript:void(0)" class="inputAssignTag multiSelectTerm btnAssign" style="display:none" data-id="addTerm"><i class="fa fa-folder-o"></i> Assign Term</a>
<a href="javascript:void(0)" class="inputAssignTag multiSelectTag assignTag btnAssign" style="display:none" data-id="addAssignTag"><i class="fa fa-plus"></i> Assign Tag</a>
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/utils/CommonViewFunction.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/utils/CommonViewFunction.js b/dashboardv2/public/js/utils/CommonViewFunction.js
index 6b8f50d..4abec08 100644
--- a/dashboardv2/public/js/utils/CommonViewFunction.js
+++ b/dashboardv2/public/js/utils/CommonViewFunction.js
@@ -398,6 +398,64 @@ define(['require', 'utils/Utils', 'modules/Modal', 'utils/Messages', 'utils/Enum
});
})
}
+ CommonViewFunction.attributeFilter = {
+ generateUrl: function(attrObj) {
+ var attrQuery = [];
+ if (attrObj) {
+ _.each(attrObj, function(obj) {
+ attrQuery.push(obj.id + "::" + obj.operator + "::" + obj.value + "::" + obj.type);
+ });
+ return attrQuery.join();
+ } else {
+ return null;
+ }
+ },
+ extractUrl: function(urlObj) {
+ var attrObj = [];
+ if (urlObj && urlObj.length) {
+ _.each(urlObj.split(","), function(obj) {
+ var temp = obj.split("::");
+ attrObj.push({ id: temp[0], operator: temp[1], value: temp[2], type: temp[3] });
+ });
+ return attrObj;
+ } else {
+ return null;
+ }
+ },
+ generateAPIObj: function(url) {
+ if (url && url.length) {
+ var parsObj = {
+ "condition": 'AND',
+ "criterion": convertKeyAndExtractObj(this.extractUrl(url))
+ }
+ return parsObj;
+ } else {
+ return null;
+ }
+
+ function convertKeyAndExtractObj(rules) {
+ var convertObj = [];
+ _.each(rules, function(rulObj) {
+ var tempObj = {};
+ // For nested
+ // if (rulObj.rules) {
+ // tempObj = {
+ // "condition": "AND",
+ // "criterion": convertKeyAndExtractObj(rulObj.rules)
+ // }
+ // } else {
+ // }
+ tempObj = {
+ "attributeName": rulObj.id,
+ "operator": rulObj.operator,
+ "attributeValue": (rulObj.type === "date" ? Date.parse(rulObj.value) : rulObj.value)
+ }
+ convertObj.push(tempObj);
+ });
+ return convertObj;
+ }
+ }
+ }
CommonViewFunction.addRestCsrfCustomHeader = function(xhr, settings) {
// if (settings.url == null || !settings.url.startsWith('/webhdfs/')) {
if (settings.url == null) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/utils/TableLayout.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/utils/TableLayout.js b/dashboardv2/public/js/utils/TableLayout.js
index db0a9da..3c9ff3b 100644
--- a/dashboardv2/public/js/utils/TableLayout.js
+++ b/dashboardv2/public/js/utils/TableLayout.js
@@ -238,7 +238,9 @@ define(['require',
*/
renderTable: function() {
var that = this;
- this.rTableList.show(new Backgrid.Grid(this.gridOpts));
+ this.rTableList.show(new Backgrid.Grid(this.gridOpts).on('backgrid:rendered', function() {
+ that.trigger('backgrid:rendered', this)
+ }));
},
/**
@@ -392,4 +394,4 @@ define(['require',
});
return FSTableLayout;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js b/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
index 8f41399..da42554 100644
--- a/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
+++ b/dashboardv2/public/js/views/business_catalog/SideNavLayoutView.js
@@ -66,7 +66,7 @@ define(['require',
return events;
},
initialize: function(options) {
- _.extend(this, _.pick(options, 'url', 'value', 'tag', 'selectFirst', 'classificationDefCollection', 'typeHeaders', 'searchVent', 'entityDefCollection', 'enumDefCollection', 'filterObj'));
+ _.extend(this, _.pick(options, 'url', 'value', 'tag', 'selectFirst', 'classificationDefCollection', 'typeHeaders', 'searchVent', 'entityDefCollection', 'enumDefCollection'));
if (Globals.taxonomy) {
this.tabClass = "tab col-sm-4";
} else {
@@ -97,7 +97,6 @@ define(['require',
collection: that.classificationDefCollection,
tag: that.tag,
value: that.value,
- filterObj: that.filterObj,
typeHeaders: that.typeHeaders
}));
});
@@ -107,7 +106,6 @@ define(['require',
require(['views/search/SearchLayoutView'], function(SearchLayoutView) {
that.RSearchLayoutView.show(new SearchLayoutView({
value: that.value,
- filterObj: that.filterObj,
searchVent: that.searchVent,
typeHeaders: that.typeHeaders,
entityDefCollection: that.entityDefCollection,
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/search/QueryBuilderView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/QueryBuilderView.js b/dashboardv2/public/js/views/search/QueryBuilderView.js
index ec20734..e8acf18 100644
--- a/dashboardv2/public/js/views/search/QueryBuilderView.js
+++ b/dashboardv2/public/js/views/search/QueryBuilderView.js
@@ -20,9 +20,10 @@ define(['require',
'backbone',
'hbs!tmpl/search/QueryBuilder_tmpl',
'utils/Utils',
+ 'utils/CommonViewFunction',
'query-builder',
'daterangepicker'
-], function(require, Backbone, QueryBuilder_Tmpl, Utils) {
+], function(require, Backbone, QueryBuilder_Tmpl, Utils, CommonViewFunction) {
var QueryBuilderView = Backbone.Marionette.LayoutView.extend(
/** @lends QueryBuilderView */
@@ -51,8 +52,9 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'attrObj', 'value', 'typeHeaders', 'filterObj', 'entityDefCollection', 'enumDefCollection', 'tag'));
+ _.extend(this, _.pick(options, 'attrObj', 'value', 'typeHeaders', 'entityDefCollection', 'enumDefCollection', 'tag'));
this.attrObj = _.sortBy(this.attrObj, 'name');
+ this.filterType = this.tag ? 'tagFilters' : 'entityFilters';
},
bindEvents: function() {},
getOperator: function(type) {
@@ -89,8 +91,8 @@ define(['require',
format: 'MM/DD/YYYY h:mm A'
}
};
- if (rules && rules.rules) {
- var valueObj = _.find(rules.rules, { id: obj.id });
+ if (rules) {
+ var valueObj = _.find(rules, { id: obj.id });
if (valueObj) {
obj.plugin_config["startDate"] = valueObj.value;
}
@@ -128,15 +130,8 @@ define(['require',
onRender: function() {
var that = this,
filters = [];
- if (this.filterObj) {
- var filter = this.filterObj[(this.tag ? 'tagFilters' : 'entityFilters')],
- tagTermName = this.tag ? this.value.tag : this.value.type;
- if (filter) {
- ruleObj = filter[tagTermName];
- if (ruleObj) {
- var rules_widgets = ruleObj.rule;
- }
- }
+ if (this.value) {
+ var rules_widgets = CommonViewFunction.attributeFilter.extractUrl(this.value[this.filterType]);
}
_.each(this.attrObj, function(obj) {
var returnObj = that.getObjDef(obj, rules_widgets);
@@ -145,20 +140,6 @@ define(['require',
}
});
filters = _.uniq(filters, 'id');
- if (rules_widgets) {
- for (var i = 0; i < rules_widgets.rules.length; i++) {
- if (!_.find(filters, { id: rules_widgets.rules[i].id })) {
- var type = (this.tag ? 'tagFilters' : 'entityFilters');
- var list = JSON.parse(Utils.localStorage.getValue(type));
- delete list[this.value.tag];
- list = _.isEmpty(list) ? null : list;
- Utils.localStorage.setValue(type, JSON.stringify(list));
- this.filterObj[type] = list;
- rules_widgets = null;
- break;
- }
- }
- }
if (filters && !_.isEmpty(filters)) {
this.ui.builder.queryBuilder({
plugins: ['bt-tooltip-errors'],
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/search/SearchDetailLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchDetailLayoutView.js b/dashboardv2/public/js/views/search/SearchDetailLayoutView.js
index 32ad542..b22fae8 100644
--- a/dashboardv2/public/js/views/search/SearchDetailLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchDetailLayoutView.js
@@ -43,7 +43,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'value', 'initialView', 'entityDefCollection', 'typeHeaders', 'searchVent', 'enumDefCollection', 'filterObj'));
+ _.extend(this, _.pick(options, 'value', 'initialView', 'entityDefCollection', 'typeHeaders', 'searchVent', 'enumDefCollection'));
},
bindEvents: function() {},
onRender: function() {
@@ -62,7 +62,6 @@ define(['require',
entityDefCollection: that.entityDefCollection,
typeHeaders: that.typeHeaders,
searchVent: that.searchVent,
- filterObj: that.filterObj,
enumDefCollection: that.enumDefCollection
}));
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/search/SearchLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchLayoutView.js b/dashboardv2/public/js/views/search/SearchLayoutView.js
index 76d6f77..0c39b02 100644
--- a/dashboardv2/public/js/views/search/SearchLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchLayoutView.js
@@ -22,7 +22,8 @@ define(['require',
'utils/Utils',
'utils/UrlLinks',
'utils/Globals',
-], function(require, Backbone, SearchLayoutViewTmpl, Utils, UrlLinks, Globals) {
+ 'utils/CommonViewFunction'
+], function(require, Backbone, SearchLayoutViewTmpl, Utils, UrlLinks, Globals, CommonViewFunction) {
'use strict';
var SearchLayoutView = Backbone.Marionette.LayoutView.extend(
@@ -80,7 +81,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'value', 'typeHeaders', 'searchVent', 'entityDefCollection', 'enumDefCollection', 'classificationDefCollection', 'filterObj'));
+ _.extend(this, _.pick(options, 'value', 'typeHeaders', 'searchVent', 'entityDefCollection', 'enumDefCollection', 'classificationDefCollection'));
this.type = "basic";
var param = Utils.getUrlState.getQueryParams();
this.query = {
@@ -121,52 +122,88 @@ define(['require',
this.checkForButtonVisiblity();
}, this);
},
- bindSelect2Events: function(argument) {
- var that = this;
- this.ui.typeLov.on('select2:select', function(argument) {
- // this function calles after checkForButtonVisiblity that is why disabled flter here
- that.ui.typeAttrFilter.prop('disabled', false);
- _.extend(that.value, { 'type': this.value });
- that.makeFilterButtonActive('type');
- });
- this.ui.tagLov.on('select2:select', function(argument) {
- // this function calles after checkForButtonVisiblity that is why disabled flter here
- that.ui.tagAttrFilter.prop('disabled', false);
- _.extend(that.value, { 'tag': this.value });
- that.makeFilterButtonActive('tag');
- });
- this.ui.typeLov.on('select2:unselect', function(argument) {
- _.extend(that.value, { 'type': null });
- });
- this.ui.tagLov.on('select2:unselect', function(argument) {
- _.extend(that.value, { 'tag': null });
- });
- },
- makeFilterButtonActive: function(type) {
- if (this.filterObj) {
- var tagFilters = this.filterObj.tagFilters,
- entityFilters = this.filterObj.entityFilters;
- if (type == "type") {
- if (_.has(entityFilters, this.value[type])) {
- this.query[this.type]['entityFilters'] = +new Date();
- this.ui.typeAttrFilter.addClass('active');
+ makeFilterButtonActive: function(filtertypeParam) {
+ var filtertype = ['entityFilters', 'tagFilters'],
+ that = this;
+ if (filtertypeParam) {
+ if (_.isArray(filtertypeParam)) {
+ filtertype = filtertypeParam;
+ } else if (_.isString(filtertypeParam)) {
+ filtertype = [filtertypeParam];
+ }
+ }
+ var typeCheck = function(filterQueryObj, type) {
+ var filterObj = filterQueryObj[type];
+ if (that.value.type) {
+ if (filterObj && filterObj.length) {
+ that.ui.typeAttrFilter.addClass('active');
} else {
- this.query[this.type]['entityFilters'] = null;
- this.ui.typeAttrFilter.removeClass('active');
+ filterQueryObj[type] = null;
+ that.value[type] = null;
+ that.ui.typeAttrFilter.removeClass('active');
}
+ that.ui.typeAttrFilter.prop('disabled', false);
+ } else {
+ filterQueryObj[type] = null;
+ that.value[type] = null;
+ that.ui.typeAttrFilter.removeClass('active');
+ that.ui.typeAttrFilter.prop('disabled', true);
}
- if (type == "tag") {
- if (_.has(tagFilters, this.value[type])) {
- this.query[this.type]['tagFilters'] = +new Date();
- this.ui.tagAttrFilter.addClass('active');
+
+ }
+ var tagCheck = function(filterQueryObj, type) {
+ var filterObj = filterQueryObj[type];
+ if (that.value.tag) {
+ that.ui.tagAttrFilter.prop('disabled', false);
+ if (filterObj && filterObj.length) {
+ that.ui.tagAttrFilter.addClass('active');
} else {
- this.query[this.type]['tagFilters'] = null;
- this.ui.tagAttrFilter.removeClass('active');
+ filterQueryObj[type] = null;
+ that.value[type] = null;
+ that.ui.tagAttrFilter.removeClass('active');
}
+ } else {
+ filterQueryObj[type] = null;
+ that.value[type] = null;
+ that.ui.tagAttrFilter.removeClass('active');
+ that.ui.tagAttrFilter.prop('disabled', true);
}
}
+ _.each(filtertype, function(type) {
+ var filterObj = that.query[that.type][type],
+ filterQueryObj = that.query[that.type];
+ if (type == "entityFilters") {
+ typeCheck(filterQueryObj, type)
+ }
+ if (type == "tagFilters") {
+ tagCheck(filterQueryObj, type)
+ }
+ });
},
- checkForButtonVisiblity: function() {
+ checkForButtonVisiblity: function(e) {
+ if (this.type == "basic" && e && e.currentTarget) {
+ var $el = $(e.currentTarget),
+ isTagEl = $el.data('id') == "tagLOV" ? true : false;
+ if (e.type == "change" && $el.select2('data')) {
+ var value = $el.val(),
+ key = (isTagEl ? 'tag' : 'type'),
+ filterType = (isTagEl ? 'tagFilters' : 'entityFilters'),
+ value = value.length ? value : null;
+ if (this.value) {
+ if (this.value[key] !== value || (!value && !this.value[key]) || (!this.value[filterType])) {
+ var temp = {};
+ temp[key] = value;
+ _.extend(this.value, temp);
+ this.query[this.type][filterType] = null;
+ this.value[filterType] = null;
+ this.makeFilterButtonActive(filterType);
+ }
+ } else {
+ this.ui.tagAttrFilter.prop('disabled', true);
+ this.ui.typeAttrFilter.prop('disabled', true);
+ }
+ }
+ }
var that = this,
value = this.ui.searchInput.val() || this.ui.typeLov.val();
if (!this.dsl && !value) {
@@ -180,24 +217,6 @@ define(['require',
} else {
this.ui.searchBtn.attr("disabled", "true");
}
- if (this.value) {
- if (this.value.tag) {
- this.ui.tagAttrFilter.prop('disabled', false);
- } else {
- this.ui.tagAttrFilter.prop('disabled', true);
- }
- if (this.value.type) {
- this.ui.typeAttrFilter.prop('disabled', false);
- } else {
- this.ui.typeAttrFilter.prop('disabled', true);
- }
- this.makeFilterButtonActive('type');
- this.makeFilterButtonActive('tag');
- } else {
- this.ui.tagAttrFilter.prop('disabled', true);
- this.ui.typeAttrFilter.prop('disabled', true);
- }
-
},
onRender: function() {
// array of tags which is coming from url
@@ -211,8 +230,6 @@ define(['require',
placeholder: "Select",
allowClear: true
});
- this.bindSelect2Events();
- this.checkForButtonVisiblity();
},
updateQueryObject: function(param) {
if (param && param.searchType) {
@@ -270,78 +287,24 @@ define(['require',
typeHeaders: that.typeHeaders,
entityDefCollection: that.entityDefCollection,
enumDefCollection: that.enumDefCollection,
- filterObj: that.filterObj,
classificationDefCollection: that.classificationDefCollection
});
- that.attrModal.on('ok', function(e) {
- that.okAttrFilterButton();
+ that.attrModal.on('ok', function(scope, e) {
+ that.okAttrFilterButton(e);
});
});
},
- okAttrFilterButton: function() {
+ okAttrFilterButton: function(e) {
var filtertype = this.attrModal.tag ? 'tagFilters' : 'entityFilters',
- rule = this.attrModal.RQueryBuilder.currentView.ui.builder.queryBuilder('getRules'),
- result = this.getQueryBuilderParsData(rule);
-
- if (result) {
- if (!_.isEmpty(result.criterion)) {
- this.query[this.type][filtertype] = +new Date();
- if (result) {
- var filterObj = this.filterObj ? this.filterObj[filtertype] : null;
- if (!filterObj) {
- filterObj = {};
- }
- var temp = {}; // IE fix
- temp[(this.attrModal.tag ? this.value.tag : this.value.type)] = { 'result': result, 'rule': rule };
- _.extend(filterObj, temp);
- this.filterObj[filtertype] = filterObj;
- this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
- Utils.localStorage.setValue((filtertype), JSON.stringify(filterObj));
- } else {
- this.filterObj[filtertype] = null;
- this.query[this.type][filtertype] = null;
- this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
- Utils.localStorage.removeValue(filtertype);
- }
-
- }
+ rule = this.attrModal.RQueryBuilder.currentView.ui.builder.queryBuilder('getRules');
+ if (rule) {
+ this.query[this.type][filtertype] = CommonViewFunction.attributeFilter.generateUrl(rule.rules);
+ this.makeFilterButtonActive(filtertype);
this.attrModal.modal.close();
- } else {
- this.filterObj[filtertype] = null;
- this.query[this.type][filtertype] = null;
- this.makeFilterButtonActive(this.attrModal.tag ? 'tag' : 'type');
- Utils.localStorage.removeValue(filtertype);
- }
- },
- getQueryBuilderParsData: function(obj) {
- if (obj) {
- var parsObj = {
- "condition": obj.condition,
- "criterion": convertKeyAndExtractObj(obj.rules)
+ if ($(e.currentTarget).hasClass('search')) {
+ this.findSearchResult();
}
}
-
- function convertKeyAndExtractObj(rules) {
- var convertObj = [];
- _.each(rules, function(rulObj) {
- var tempObj = {}
- if (rulObj.rules) {
- tempObj = {
- "condition": rulObj.condition,
- "criterion": convertKeyAndExtractObj(rulObj.rules)
- }
- } else {
- tempObj = {
- "attributeName": rulObj.id,
- "operator": rulObj.operator,
- "attributeValue": (rulObj.type === "date" ? Date.parse(rulObj.value) : rulObj.value)
- }
- }
- convertObj.push(tempObj);
- });
- return convertObj;
- }
- return parsObj;
},
manualRender: function(paramObj) {
this.updateQueryObject(paramObj);
@@ -408,7 +371,6 @@ define(['require',
setTimeout(function() {
that.ui.searchInput.focus();
}, 0);
- //this.searchVent.trigger('searchAttribute', this.value);
}
},
findSearchResult: function() {
@@ -482,10 +444,6 @@ define(['require',
this.ui.tagLov.val("").trigger("change");
this.ui.searchInput.val("");
this.checkForButtonVisiblity();
- Utils.localStorage.removeValue('tagFilters');
- Utils.localStorage.removeValue('entityFilters');
- this.filterObj.tagFilters = null;
- this.filterObj.entityFilters = null;
Utils.setUrl({
url: '#!/search/searchResult',
urlParams: {
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/search/SearchQueryView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchQueryView.js b/dashboardv2/public/js/views/search/SearchQueryView.js
index 37ccf90..281e37e 100644
--- a/dashboardv2/public/js/views/search/SearchQueryView.js
+++ b/dashboardv2/public/js/views/search/SearchQueryView.js
@@ -50,17 +50,27 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'value', 'entityDefCollection', 'typeHeaders', 'searchVent', 'enumDefCollection', 'classificationDefCollection', 'tag', 'filterObj'));
+ _.extend(this, _.pick(options, 'value', 'entityDefCollection', 'typeHeaders', 'searchVent', 'enumDefCollection', 'classificationDefCollection', 'tag'));
this.bindEvents();
var that = this;
this.modal = new Modal({
title: 'Attribute Filter',
content: this,
- okText: 'Apply',
- cancelText: "Cancel",
allowCancel: true,
okCloses: false,
- width: '50%'
+ width: '50%',
+ buttons: [{
+ text: 'Apply',
+ btnClass: "ok"
+ },
+ {
+ text: 'Apply & Search',
+ btnClass: "ok search"
+ }, {
+ text: 'Cancel',
+ btnClass: "cancel"
+ }
+ ]
}).open();
this.modal.on('closeModal', function() {
that.modal.trigger('cancel');
@@ -73,7 +83,6 @@ define(['require',
searchVent: this.searchVent,
entityDefCollection: this.entityDefCollection,
enumDefCollection: this.enumDefCollection,
- filterObj: this.filterObj,
classificationDefCollection: this.classificationDefCollection
}
@@ -110,4 +119,4 @@ define(['require',
}
});
return SearchQueryView;
-});
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/search/SearchResultLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/search/SearchResultLayoutView.js b/dashboardv2/public/js/views/search/SearchResultLayoutView.js
index 319beab..d1b1a02 100644
--- a/dashboardv2/public/js/views/search/SearchResultLayoutView.js
+++ b/dashboardv2/public/js/views/search/SearchResultLayoutView.js
@@ -62,9 +62,7 @@ define(['require',
editEntityButton: "[data-id='editEntityButton']",
createEntity: "[data-id='createEntity']",
checkDeletedEntity: "[data-id='checkDeletedEntity']",
- containerCheckBox: "[data-id='containerCheckBox']",
- filterPanel: "#filterPanel",
- filterQuery: "#filterQuery"
+ containerCheckBox: "[data-id='containerCheckBox']"
},
templateHelpers: function() {
return {
@@ -129,8 +127,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'value', 'initialView', 'entityDefCollection', 'typeHeaders', 'searchVent', 'enumDefCollection', 'tagCollection', 'filterObj'));
- var pagination = "";
+ _.extend(this, _.pick(options, 'value', 'initialView', 'entityDefCollection', 'typeHeaders', 'searchVent', 'enumDefCollection', 'tagCollection'));
this.entityModel = new VEntity();
this.searchCollection = new VSearchList();
this.limit = 25;
@@ -142,7 +139,8 @@ define(['require',
includePagination: false,
includeFooterRecords: false,
includeColumnManager: (this.value && this.value.searchType === "basic" ? true : false),
- includeOrderAbleColumns: false,
+ includeOrderAbleColumns: true,
+ includeSizeAbleColumns: true,
columnOpts: {
opts: {
initialColumnsVisible: null,
@@ -237,9 +235,6 @@ define(['require',
this.listenTo(this.searchVent, "search:refresh", function(model, response) {
this.fetchCollection();
}, this);
- this.listenTo(this.searchVent, "searchAttribute", function(obj) {
- this.showHideFilter(obj);
- }, this);
},
onRender: function() {
if (!this.initialView) {
@@ -269,76 +264,66 @@ define(['require',
this.$(".entityLink").show();
}
}
- this.showHideFilter();
},
updateColumnList: function(updatedList) {
if (updatedList) {
var listOfColumns = []
_.map(updatedList, function(obj) {
var key = obj.name;
- if (key == "selected" || key == "displayText" || key == "description" || key == "typeName" || key == "owner" || key == "tag" || key == "terms") {
- return;
- }
if (obj.renderable) {
listOfColumns.push(obj.name);
}
});
+ listOfColumns = _.sortBy(listOfColumns);
this.value.attributes = listOfColumns.length ? listOfColumns.join(",") : null;
}
this.columnToShow = this.value && this.value.attributes ? this.value.attributes.split(',') : [];
},
generateQueryOfFilter: function() {
var value = this.value,
- entityFilters = this.filterObj && this.filterObj.entityFilters ? this.filterObj.entityFilters[value.type] : null,
- tagFilters = this.filterObj && this.filterObj.tagFilters ? this.filterObj.tagFilters[value.tag] : null,
+ entityFilters = CommonViewFunction.attributeFilter.extractUrl(value.entityFilters),
+ tagFilters = CommonViewFunction.attributeFilter.extractUrl(value.tagFilters),
queryArray = [],
objToString = function(filterObj) {
var tempObj = [];
- _.each(filterObj.rules, function(obj) {
- tempObj.push('<span class="key">' + obj.field + '</span> <span class="operator">' + obj.operator + '</span> <span class="value">' + obj.value + "</span>")
+ _.each(filterObj, function(obj) {
+ tempObj.push('<span class="key">' + obj.id + '</span> <span class="operator">' + obj.operator + '</span> <span class="value">' + obj.value + "</span>")
});
return tempObj.join(' <span class="operator">AND</span> ');
}
- if (entityFilters) {
- var typeKeyValue = '<span class="key">Type:</span> <span class="value">' + value.type + '</span> <span class="operator">AND</span> ';
- queryArray = queryArray.concat(typeKeyValue + objToString(entityFilters.rule));
+ if (value.type) {
+ var typeKeyValue = '<span class="key">Type:</span> <span class="value">' + value.type + '</span>';
+ if (entityFilters) {
+ typeKeyValue += ' <span class="operator">AND</span> ' + objToString(entityFilters);
+ }
+ queryArray.push(typeKeyValue)
}
- if (tagFilters) {
- var tagKeyValue = '<span class="key">Tag:</span> <span class="value">' + value.tag + '</span> <span class="operator">AND</span> ';
- queryArray = queryArray.concat(tagKeyValue + objToString(tagFilters.rule));
-
+ if (value.tag) {
+ var tagKeyValue = '<span class="key">Tag:</span> <span class="value">' + value.tag + '</span>';
+ if (tagFilters) {
+ tagKeyValue += ' <span class="operator">AND</span> ' + objToString(tagFilters);
+ }
+ queryArray.push(tagKeyValue);
}
- if (queryArray.length == 2) {
- return "<span>(</span> " + queryArray.join('<span> )</span> <span>AND</span> <span>(</span> ') + " <span>)</span>";
- } else {
- return queryArray.join();
+ if (value.query) {
+ queryArray.push('<span class="key">Query:</span> <span class="value">' + value.query + '</span> ');
}
- },
- showHideFilter: function() {
- if (this.value) {
- if (Utils.getUrlState.isSearchTab() && this.value.searchType == "basic") {
- var query = this.generateQueryOfFilter();
- if (query) {
- this.ui.filterQuery.html(query);
- this.ui.filterPanel.show();
- } else
- this.ui.filterPanel.hide();
- } else {
- this.ui.filterPanel.hide();
- }
+ if (queryArray.length == 1) {
+ return queryArray.join();
} else {
- this.ui.filterPanel.hide();
+ return "<span>(</span> " + queryArray.join('<span> )</span> <span>AND</span> <span>(</span> ') + " <span>)</span>";
+
}
},
fetchCollection: function(value, clickObj) {
var that = this,
isPostMethod = this.value.searchType === "basic" && Utils.getUrlState.isSearchTab(),
- tagFilters = this.filterObj && this.filterObj.tagFilters ? this.filterObj.tagFilters[this.value.tag] : null,
- entityFilters = this.filterObj && this.filterObj.entityFilters ? this.filterObj.entityFilters[this.value.type] : null,
+ tagFilters = CommonViewFunction.attributeFilter.generateAPIObj(this.value.tagFilters),
+ entityFilters = CommonViewFunction.attributeFilter.generateAPIObj(this.value.entityFilters),
filterObj = {
- 'entityFilters': entityFilters ? entityFilters.result : null,
- 'tagFilters': tagFilters ? tagFilters.result : null,
- 'attributes': this.columnToShow.length ? this.columnToShow : null
+ 'entityFilters': entityFilters,
+ 'tagFilters': tagFilters,
+ 'attributes': this.columnToShow.length ? _.without(this.columnToShow, "selected", "name", "description", "typeName", "owner", "tag", "terms") : null
}
this.showLoader();
if (Globals.searchApiCallRef && Globals.searchApiCallRef.readyState === 1) {
@@ -393,7 +378,7 @@ define(['require',
if (that.searchCollection.queryParams.query) {
resultArr.push(that.searchCollection.queryParams.query)
}
- var searchString = 'Results for <b>' + _.escape(resultArr.join(that.searchType == 'Advanced Search' ? " " : " & ")) + '</b>';
+ var searchString = 'Results for: <span class="filterQuery">' + that.generateQueryOfFilter() + "</span>";
if (Globals.entityCreate && Globals.entityTypeConfList && Utils.getUrlState.isSearchTab()) {
searchString += "<p>If you do not find the entity in search result below then you can" + '<a href="javascript:void(0)" data-id="createEntity"> create new entity</a></p>';
}
@@ -412,7 +397,6 @@ define(['require',
if (isPostMethod) {
apiObj['data'] = _.extend({}, filterObj, _.pick(this.searchCollection.queryParams, 'query', 'excludeDeletedEntities', 'limit', 'offset', 'typeName', 'classification'))
Globals.searchApiCallRef = this.searchCollection.getBasicRearchResult(apiObj);
- this.showHideFilter();
} else {
apiObj.data = null;
Globals.searchApiCallRef = this.searchCollection.fetch(apiObj);
@@ -441,13 +425,13 @@ define(['require',
count = 5;
require(['utils/TableLayout'], function(TableLayout) {
var columnCollection = Backgrid.Columns.extend({
- sortKey: "position",
+ sortKey: "displayOrder",
comparator: function(item) {
return item.get(this.sortKey) || 999;
},
setPositions: function() {
_.each(this.models, function(model, index) {
- model.set("position", index + 1, { silent: true });
+ model.set("displayOrder", index + 1, { silent: true });
});
return this;
}
@@ -491,10 +475,13 @@ define(['require',
name: "selected",
label: "Select",
cell: "select-row",
+ resizeable: false,
+ orderable: false,
+ renderable: (that.columnToShow && that.columnToShow.length ? _.contains(that.columnToShow, 'selected') : true),
headerCell: "select-all"
};
- col['displayText'] = {
+ col['name'] = {
label: "Name",
cell: "html",
editable: false,
@@ -556,7 +543,7 @@ define(['require',
sortable: false,
resizeable: true,
orderable: true,
- renderable: true,
+ renderable: (that.columnToShow && that.columnToShow.length ? _.contains(that.columnToShow, 'typeName') : true),
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
var obj = model.toJSON();
@@ -590,7 +577,7 @@ define(['require',
sortable: false,
resizeable: true,
orderable: true,
- renderable: true,
+ renderable: (that.columnToShow && that.columnToShow.length ? _.contains(that.columnToShow, 'tag') : true),
className: 'searchTag',
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
@@ -612,7 +599,7 @@ define(['require',
sortable: false,
resizeable: true,
orderable: true,
- renderable: true,
+ renderable: (that.columnToShow && that.columnToShow.length ? _.contains(that.columnToShow, 'terms') : true),
className: 'searchTerm',
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
@@ -636,8 +623,11 @@ define(['require',
var attrObj = Utils.getNestedSuperTypeObj({ data: def.toJSON(), collection: this.entityDefCollection, attrMerge: true });
_.each(attrObj, function(obj, key) {
var key = obj.name,
- isEenderable = that.columnToShow.length ? _.contains(that.columnToShow, key) : false;
+ isRenderable = _.contains(that.columnToShow, key)
if (key == "name" || key == "description" || key == "owner") {
+ if (that.columnToShow && that.columnToShow.length) {
+ col[key].renderable = isRenderable;
+ }
return;
}
col[obj.name] = {
@@ -647,7 +637,7 @@ define(['require',
sortable: false,
resizeable: true,
orderable: true,
- renderable: isEenderable,
+ renderable: isRenderable,
formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
fromRaw: function(rawValue, model) {
var modelObj = model.toJSON();
http://git-wip-us.apache.org/repos/asf/atlas/blob/f511f272/dashboardv2/public/js/views/tag/TagLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/tag/TagLayoutView.js b/dashboardv2/public/js/views/tag/TagLayoutView.js
index 2714770..5f58ad7 100644
--- a/dashboardv2/public/js/views/tag/TagLayoutView.js
+++ b/dashboardv2/public/js/views/tag/TagLayoutView.js
@@ -59,7 +59,7 @@ define(['require',
* @constructs
*/
initialize: function(options) {
- _.extend(this, _.pick(options, 'tag', 'collection', 'typeHeaders', 'filterObj', 'value'));
+ _.extend(this, _.pick(options, 'tag', 'collection', 'typeHeaders', 'value'));
},
bindEvents: function() {
var that = this;
@@ -417,15 +417,6 @@ define(['require',
}
that.collection.remove(deleteTagData);
// to update tag list of search tab fetch typeHeaders.
- var tagList = JSON.parse(Utils.localStorage.getValue('tagFilters'));
- if (tagList) {
- delete tagList[that.tagName];
- }
- tagList = _.isEmpty(tagList) ? null : tagList;
- if (that.filterObj['tagFilters'] && that.filterObj['tagFilters'][that.tagName]) {
- delete that.filterObj['tagFilters'][that.tagName];
- }
- Utils.localStorage.setValue('tagFilters', JSON.stringify(tagList));
that.typeHeaders.fetch({ reset: true });
}
});
[37/39] atlas git commit: ATLAS-1982: doc,
website updated - to remove references to 'incubator',
and with current PMC info
Posted by ma...@apache.org.
ATLAS-1982: doc, website updated - to remove references to 'incubator', and with current PMC info
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/de251913
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/de251913
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/de251913
Branch: refs/heads/feature-odf
Commit: de251913cdcfade1e77c826072d6869f78dfc91d
Parents: d23d61b
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Jul 21 18:04:11 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sun Jul 23 21:43:48 2017 -0700
----------------------------------------------------------------------
DISCLAIMER.txt | 19 +-
README.txt | 14 +-
addons/falcon-bridge-shim/pom.xml | 2 +-
addons/falcon-bridge/pom.xml | 2 +-
addons/hdfs-model/pom.xml | 2 +-
addons/hive-bridge-shim/pom.xml | 2 +-
addons/hive-bridge/pom.xml | 2 +-
addons/sqoop-bridge-shim/pom.xml | 2 +-
addons/sqoop-bridge/pom.xml | 2 +-
addons/storm-bridge-shim/pom.xml | 2 +-
addons/storm-bridge/pom.xml | 2 +-
authorization/pom.xml | 2 +-
build-tools/pom.xml | 2 +-
catalog/pom.xml | 2 +-
client/pom.xml | 2 +-
common/pom.xml | 2 +-
dashboardv2/package.json | 2 +-
dashboardv2/pom.xml | 4 +-
.../business_catalog/BusinessCatalogHeader.html | 2 +-
.../search/AdvancedSearchInfo_tmpl.html | 2 +-
.../public/js/templates/site/header.html | 2 +-
distro/pom.xml | 2 +-
docs/pom.xml | 8 +-
docs/src/site/site.xml | 14 +-
docs/src/site/twiki/InstallationSteps.twiki | 2 +-
docs/src/site/twiki/index.twiki | 8 -
graphdb/api/pom.xml | 2 +-
graphdb/common/pom.xml | 2 +-
graphdb/graphdb-impls/pom.xml | 2 +-
graphdb/pom.xml | 2 +-
graphdb/titan0/pom.xml | 2 +-
graphdb/titan1/pom.xml | 2 +-
intg/pom.xml | 2 +-
notification/pom.xml | 2 +-
plugin-classloader/pom.xml | 2 +-
pom.xml | 418 ++++++++++---------
repository/pom.xml | 2 +-
server-api/pom.xml | 2 +-
shaded/hbase-client-shaded/pom.xml | 2 +-
shaded/hbase-server-shaded/pom.xml | 2 +-
typesystem/pom.xml | 2 +-
webapp/pom.xml | 4 +-
42 files changed, 273 insertions(+), 284 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/DISCLAIMER.txt
----------------------------------------------------------------------
diff --git a/DISCLAIMER.txt b/DISCLAIMER.txt
index f577155..39ab3c1 100755
--- a/DISCLAIMER.txt
+++ b/DISCLAIMER.txt
@@ -1,16 +1,7 @@
-Apache Atlas is an effort undergoing incubation at the Apache
-Software Foundation (ASF), sponsored by the Apache Incubator PMC.
+Apache Atlas is a Top Level Project (TLP) at the Apache Software
+Foundation (ASF).
-Incubation is required of all newly accepted projects until a further
-review indicates that the infrastructure, communications, and decision
-making process have stabilized in a manner consistent with other
-successful ASF projects.
+This product includes software developed at The Apache Software
+Foundation (http://www.apache.org/).
-While incubation status is not necessarily a reflection of the
-completeness or stability of the code, it does indicate that the
-project has yet to be fully endorsed by the ASF.
-
-For more information about the incubation status of the Apache Atlas
-project you can go to the following page:
-
-http://incubator.apache.org/projects/atlas.html
\ No newline at end of file
+http://atlas.apache.org
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/README.txt
----------------------------------------------------------------------
diff --git a/README.txt b/README.txt
index 9716eb9..342520f 100755
--- a/README.txt
+++ b/README.txt
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-Apache Atlas (incubating) Overview
+Apache Atlas Overview
Apache Atlas framework is an extensible set of core
foundational governance services – enabling enterprises to effectively and
@@ -31,21 +31,13 @@ The metadata veracity is maintained by leveraging Apache Ranger to prevent
non-authorized access paths to data at runtime.
Security is both role based (RBAC) and attribute based (ABAC).
-
-Apache Atlas is an effort undergoing incubation at the Apache
-Software Foundation (ASF), sponsored by the Apache Incubator PMC.
-
-For more information about the incubation status of the Apache Atlas
-project you can go to the following page:
-http://incubator.apache.org/projects/atlas.html
-
Build Process
=============
1. Get Atlas sources to your local directory, for example with following commands
$ cd <your-local-directory>
- $ git clone https://github.com/apache/incubator-atlas.git
- $ cd incubator-atlas
+ $ git clone https://github.com/apache/atlas.git
+ $ cd atlas
# Checkout the branch or tag you would like to build
#
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/falcon-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge-shim/pom.xml b/addons/falcon-bridge-shim/pom.xml
index 7089adb..2d0459c 100755
--- a/addons/falcon-bridge-shim/pom.xml
+++ b/addons/falcon-bridge-shim/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>falcon-bridge-shim</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/falcon-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index 287b8e9..bc71925 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>falcon-bridge</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/hdfs-model/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/pom.xml b/addons/hdfs-model/pom.xml
index bb2634d..7ff085c 100644
--- a/addons/hdfs-model/pom.xml
+++ b/addons/hdfs-model/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>hdfs-model</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/hive-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge-shim/pom.xml b/addons/hive-bridge-shim/pom.xml
index cdc93c1..d87947f 100755
--- a/addons/hive-bridge-shim/pom.xml
+++ b/addons/hive-bridge-shim/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>hive-bridge-shim</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 983a04f..64ffe04 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>hive-bridge</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/sqoop-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge-shim/pom.xml b/addons/sqoop-bridge-shim/pom.xml
index a1d71de..e7cf97a 100755
--- a/addons/sqoop-bridge-shim/pom.xml
+++ b/addons/sqoop-bridge-shim/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>sqoop-bridge-shim</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/sqoop-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml
index d52167b..33c5089 100644
--- a/addons/sqoop-bridge/pom.xml
+++ b/addons/sqoop-bridge/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>sqoop-bridge</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/storm-bridge-shim/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge-shim/pom.xml b/addons/storm-bridge-shim/pom.xml
index f02dbc1..cf2a025 100755
--- a/addons/storm-bridge-shim/pom.xml
+++ b/addons/storm-bridge-shim/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>storm-bridge-shim</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/addons/storm-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml
index 5fb1be0..a4e4d0f 100644
--- a/addons/storm-bridge/pom.xml
+++ b/addons/storm-bridge/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>
<artifactId>storm-bridge</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/authorization/pom.xml
----------------------------------------------------------------------
diff --git a/authorization/pom.xml b/authorization/pom.xml
index e806fb1..cac4fd7 100644
--- a/authorization/pom.xml
+++ b/authorization/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-authorization</artifactId>
<name>Apache Atlas Authorization</name>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/build-tools/pom.xml
----------------------------------------------------------------------
diff --git a/build-tools/pom.xml b/build-tools/pom.xml
index 530ff81..454ed38 100644
--- a/build-tools/pom.xml
+++ b/build-tools/pom.xml
@@ -24,6 +24,6 @@
<artifactId>atlas-buildtools</artifactId>
<name>Apache Atlas Server Build Tools</name>
<description>Apache Atlas Build Tools like Checkstyle</description>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<packaging>jar</packaging>
</project>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/catalog/pom.xml
----------------------------------------------------------------------
diff --git a/catalog/pom.xml b/catalog/pom.xml
index 2828bed..36eeb4a 100755
--- a/catalog/pom.xml
+++ b/catalog/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-catalog</artifactId>
<description>Apache Atlas Business Catalog Module</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/client/pom.xml
----------------------------------------------------------------------
diff --git a/client/pom.xml b/client/pom.xml
index 4e72b4e..6db5ef5 100755
--- a/client/pom.xml
+++ b/client/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-client</artifactId>
<description>Apache Atlas Client</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index bcbb5c5..ddbd617 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-common</artifactId>
<description>Apache Atlas Common</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/dashboardv2/package.json
----------------------------------------------------------------------
diff --git a/dashboardv2/package.json b/dashboardv2/package.json
index bcf5fe7..e2e3d8c 100644
--- a/dashboardv2/package.json
+++ b/dashboardv2/package.json
@@ -4,7 +4,7 @@
"private": true,
"repository": {
"type": "git",
- "url": "https://git-wip-us.apache.org/repos/asf/incubator-atlas.git"
+ "url": "https://git-wip-us.apache.org/repos/asf/atlas.git"
},
"engines": {
"node": "4.4.x",
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/dashboardv2/pom.xml
----------------------------------------------------------------------
diff --git a/dashboardv2/pom.xml b/dashboardv2/pom.xml
index 5a2057d..20d58ad 100644
--- a/dashboardv2/pom.xml
+++ b/dashboardv2/pom.xml
@@ -23,14 +23,14 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-dashboardv2</artifactId>
<description>Apache Atlas UI Application</description>
<name>Apache Atlas UI</name>
<packaging>war</packaging>
- <url>http://atlas.incubator.apache.org</url>
+ <url>http://atlas.apache.org</url>
<build>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/dashboardv2/public/js/templates/business_catalog/BusinessCatalogHeader.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/templates/business_catalog/BusinessCatalogHeader.html b/dashboardv2/public/js/templates/business_catalog/BusinessCatalogHeader.html
index ae172cc..3f8928e 100644
--- a/dashboardv2/public/js/templates/business_catalog/BusinessCatalogHeader.html
+++ b/dashboardv2/public/js/templates/business_catalog/BusinessCatalogHeader.html
@@ -18,7 +18,7 @@
</ol>
<header class="clearfix">
<div class="btn-group pull-right header-menu">
- <a target="_blank" href="http://atlas.incubator.apache.org/"><i class="fa fa-question-circle"></i></a>
+ <a target="_blank" href="http://atlas.apache.org/"><i class="fa fa-question-circle"></i></a>
<a href="javascript:void(0);" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false"><i class="fa fa-user user-circle"></i><span class="userName"></span></a>
<ul class="dropdown-menu">
<li class="aboutAtlas"><a href="javascript:void(0)">About</a></li>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/dashboardv2/public/js/templates/search/AdvancedSearchInfo_tmpl.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/templates/search/AdvancedSearchInfo_tmpl.html b/dashboardv2/public/js/templates/search/AdvancedSearchInfo_tmpl.html
index 9b8e1bc..3e13fce 100644
--- a/dashboardv2/public/js/templates/search/AdvancedSearchInfo_tmpl.html
+++ b/dashboardv2/public/js/templates/search/AdvancedSearchInfo_tmpl.html
@@ -33,6 +33,6 @@
<li>Query Example</li>
</ul>
<ul>
- <a href="http://atlas.incubator.apache.org/Search.html" target="_blank">More sample queries and use-cases >></a>
+ <a href="http://atlas.apache.org/Search.html" target="_blank">More sample queries and use-cases >></a>
</ul>
</div>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/dashboardv2/public/js/templates/site/header.html
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/templates/site/header.html b/dashboardv2/public/js/templates/site/header.html
index 558dd67..4673688 100644
--- a/dashboardv2/public/js/templates/site/header.html
+++ b/dashboardv2/public/js/templates/site/header.html
@@ -16,7 +16,7 @@
-->
<header class="clearfix">
<div class="btn-group pull-right header-menu ">
- <a target="_blank" href="http://atlas.incubator.apache.org/"><i class="fa fa-question-circle"></i></a>
+ <a target="_blank" href="http://atlas.apache.org/"><i class="fa fa-question-circle"></i></a>
<a href="javascript:void(0);" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false" class="user-dropdown"><i class="fa fa-user user-circle "></i><span class="userName"></span></a>
<ul class="dropdown-menu">
<li class="aboutAtlas"><a href="javascript:void(0)">About</a></li>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/distro/pom.xml
----------------------------------------------------------------------
diff --git a/distro/pom.xml b/distro/pom.xml
index 3ffaee9..49ad813 100644
--- a/distro/pom.xml
+++ b/distro/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>atlas-distro</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index 018904d..15d54dd 100755
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-docs</artifactId>
<description>Apache Atlas Documentation</description>
@@ -45,9 +45,9 @@
<skip>${skipDocs}</skip>
<dependencyDetailsEnabled>false</dependencyDetailsEnabled>
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
- <webAccessUrl>https://github.com/apache/incubator-atlas.git</webAccessUrl>
- <anonymousConnection>scm:git://git.apache.org/incubator-atlas.git</anonymousConnection>
- <developerConnection>scm:https://git-wip-us.apache.org/repos/asf/incubator-atlas.git</developerConnection>
+ <webAccessUrl>https://github.com/apache/atlas.git</webAccessUrl>
+ <anonymousConnection>scm:git://git.apache.org/atlas.git</anonymousConnection>
+ <developerConnection>scm:https://git-wip-us.apache.org/repos/asf/atlas.git</developerConnection>
</configuration>
<reportSets>
<reportSet>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/docs/src/site/site.xml
----------------------------------------------------------------------
diff --git a/docs/src/site/site.xml b/docs/src/site/site.xml
index bf7c0c9..a532286 100755
--- a/docs/src/site/site.xml
+++ b/docs/src/site/site.xml
@@ -36,7 +36,7 @@
<leftColumnClass>span2</leftColumnClass>
<bodyColumnClass>span10</bodyColumnClass>
<googleSearch>
- <sitesearch>http://atlas.incubator.apache.org</sitesearch>
+ <sitesearch>http://atlas.apache.org</sitesearch>
</googleSearch>
<facebookLike/>
<googlePlusOne/>
@@ -46,17 +46,11 @@
<bannerLeft>
<name>Apache Atlas</name>
<src>./images/atlas-logo.png</src>
- <href>http://atlas.incubator.apache.org</href>
+ <href>http://atlas.apache.org</href>
<width>200px</width>
<height>45px</height>
</bannerLeft>
- <bannerRight>
- <name>Apache Incubator</name>
- <src>./images/apache-incubator-logo.png</src>
- <href>http://incubator.apache.org</href>
- </bannerRight>
-
<publishDate position="right"/>
<version position="right"/>
@@ -76,7 +70,7 @@
<item name="About" href="index.html"/>
<item name="Wiki" href="https://cwiki.apache.org/confluence/display/ATLAS"/>
<item name="News" href="https://cwiki.apache.org/confluence/display/ATLAS"/>
- <item name="Git" href="https://git-wip-us.apache.org/repos/asf/incubator-atlas.git" />
+ <item name="Git" href="https://git-wip-us.apache.org/repos/asf/atlas.git" />
<item name="Jira" href="https://issues.apache.org/jira/browse/ATLAS" />
<item name="Powered by" href="https://cwiki.apache.org/confluence/display/ATLAS/PoweredBy" />
<item name="Blog" href="http://blogs.apache.org/atlas/" />
@@ -94,7 +88,7 @@
<menu name="Releases">
<item name="0.8-incubating"
- href="http://www.apache.org/dyn/closer.cgi/incubator/atlas/0.8.0-incubating/"/>
+ href="http://www.apache.org/dyn/closer.cgi/atlas/0.8.0-incubating/"/>
<item name="0.7.1-incubating"
href="http://archive.apache.org/dist/incubator/atlas/0.7.1-incubating/"/>
<item name="0.7-incubating"
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 4869f9f..0934495 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -3,7 +3,7 @@
---+++ Building Atlas
<verbatim>
-git clone https://git-wip-us.apache.org/repos/asf/incubator-atlas.git atlas
+git clone https://git-wip-us.apache.org/repos/asf/atlas.git atlas
cd atlas
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/docs/src/site/twiki/index.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/index.twiki b/docs/src/site/twiki/index.twiki
index 82d27c0..a8e7de9 100755
--- a/docs/src/site/twiki/index.twiki
+++ b/docs/src/site/twiki/index.twiki
@@ -66,11 +66,3 @@ allows integration with the whole enterprise data ecosystem.
---+ Licensing Information
Atlas is distributed under [[http://www.apache.org/licenses/LICENSE-2.0][Apache License 2.0]].
-
----+ Disclaimer
-
-Apache Atlas is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache
-Incubator. Incubation is required of all newly accepted projects until a further review indicates that the
-infrastructure, communications, and decision making process have stabilized in a manner consistent with other
-successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability
-of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/graphdb/api/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/api/pom.xml b/graphdb/api/pom.xml
index f082023..3d6af83 100644
--- a/graphdb/api/pom.xml
+++ b/graphdb/api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-graphdb-api</artifactId>
<description>Apache Atlas Graph Datbase API</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/graphdb/common/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/common/pom.xml b/graphdb/common/pom.xml
index 8b95f36..9a3dd7c 100644
--- a/graphdb/common/pom.xml
+++ b/graphdb/common/pom.xml
@@ -18,7 +18,7 @@ under the License. -->
<parent>
<artifactId>atlas-graphdb</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-graphdb-common</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/graphdb/graphdb-impls/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/graphdb-impls/pom.xml b/graphdb/graphdb-impls/pom.xml
index feafe74..c07ce13 100644
--- a/graphdb/graphdb-impls/pom.xml
+++ b/graphdb/graphdb-impls/pom.xml
@@ -22,7 +22,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>atlas-graphdb-impls</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/graphdb/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/pom.xml b/graphdb/pom.xml
index 2112938..8ffe96d 100644
--- a/graphdb/pom.xml
+++ b/graphdb/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-graphdb</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/graphdb/titan0/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/titan0/pom.xml b/graphdb/titan0/pom.xml
index ceda1bb..d3353a7 100644
--- a/graphdb/titan0/pom.xml
+++ b/graphdb/titan0/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>atlas-graphdb</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-graphdb-titan0</artifactId>
<description>Apache Atlas Titan 0.5.4 Graph DB Impl</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/graphdb/titan1/pom.xml
----------------------------------------------------------------------
diff --git a/graphdb/titan1/pom.xml b/graphdb/titan1/pom.xml
index fc0b2f6..fb97cf9 100644
--- a/graphdb/titan1/pom.xml
+++ b/graphdb/titan1/pom.xml
@@ -23,7 +23,7 @@
<parent>
<artifactId>atlas-graphdb</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-graphdb-titan1</artifactId>
<description>Apache Atlas Titan 1.0.0 Graph DB Impl</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/intg/pom.xml
----------------------------------------------------------------------
diff --git a/intg/pom.xml b/intg/pom.xml
index 2a06ea9..ffc468f 100644
--- a/intg/pom.xml
+++ b/intg/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-intg</artifactId>
<description>Apache Atlas Integration</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/notification/pom.xml
----------------------------------------------------------------------
diff --git a/notification/pom.xml b/notification/pom.xml
index daa5d11..f780538 100644
--- a/notification/pom.xml
+++ b/notification/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-notification</artifactId>
<description>Apache Atlas Notification</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/plugin-classloader/pom.xml
----------------------------------------------------------------------
diff --git a/plugin-classloader/pom.xml b/plugin-classloader/pom.xml
index 52dacac..3f0acf9 100644
--- a/plugin-classloader/pom.xml
+++ b/plugin-classloader/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-plugin-classloader</artifactId>
<description>Apache Atlas Plugin Classloader</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ebc07e2..b7f5ea0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -28,11 +28,11 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<description>Metadata Management and Data Governance Platform over Hadoop</description>
<name>apache-atlas</name>
<packaging>pom</packaging>
- <url>http://atlas.incubator.apache.org</url>
+ <url>http://atlas.apache.org</url>
<prerequisites>
<maven>3.0.4</maven>
@@ -65,123 +65,105 @@
<mailingLists>
<mailingList>
<name>atlas-dev</name>
- <subscribe>dev-subscribe@atlas.incubator.apache.org</subscribe>
- <unsubscribe>dev-unsubscribe@atlas.incubator.apache.org</unsubscribe>
- <post>dev@atlas.incubator.apache.org</post>
+ <subscribe>dev-subscribe@atlas.apache.org</subscribe>
+ <unsubscribe>dev-unsubscribe@atlas.apache.org</unsubscribe>
+ <post>dev@atlas.apache.org</post>
<archive>http://mail-archives.apache.org/mod_mbox/atlas-dev/</archive>
</mailingList>
<mailingList>
+ <name>atlas-user</name>
+ <subscribe>user-subscribe@atlas.apache.org</subscribe>
+ <unsubscribe>user-unsubscribe@atlas.apache.org</unsubscribe>
+ <post>user@atlas.apache.org</post>
+ <archive>http://mail-archives.apache.org/mod_mbox/atlas-user/</archive>
+ </mailingList>
+ <mailingList>
<name>atlas-commits</name>
- <subscribe>commits-subscribe@atlas.incubator.apache.org</subscribe>
- <unsubscribe>commits-unsubscribe@atlas.incubator.apache.org</unsubscribe>
- <post>commits@atlas.incubator.apache.org</post>
+ <subscribe>commits-subscribe@atlas.apache.org</subscribe>
+ <unsubscribe>commits-unsubscribe@atlas.apache.org</unsubscribe>
+ <post>commits@atlas.apache.org</post>
<archive>http://mail-archives.apache.org/mod_mbox/atlas-commits/</archive>
</mailingList>
</mailingLists>
<scm>
- <connection>scm:git:git://git.apache.org/incubator-atlas.git</connection>
- <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-atlas.git</developerConnection>
+ <connection>scm:git:git://git.apache.org/atlas.git</connection>
+ <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/atlas.git</developerConnection>
<tag>HEAD</tag>
- <url>https://github.com/apache/incubator-atlas.git</url>
+ <url>https://github.com/apache/atlas.git</url>
</scm>
<developers>
<developer>
- <id>venkatesh</id>
- <name>Venkatesh Seetharam</name>
- <email>venkatesh@apache.org</email>
- <timezone>America/Los_Angeles</timezone>
+ <id>adossett</id>
+ <name>Aaron Dossett</name>
+ <email>adossett@target.com</email>
+ <timezone>America/Denver</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Hortonworks Inc.</organization>
- <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
+ <organization>Target Inc.</organization>
+ <!--<organizationUrl>http://www.target.com</organizationUrl>-->
</developer>
<developer>
- <id>rhbutani</id>
- <name>Harish Butani</name>
- <email>rhbutani@apache.org</email>
+ <id>AAhn</id>
+ <name>Andrew Ahn</name>
+ <email>aa@apache.org</email>
<timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization />
- <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
+ <organization></organization>
</developer>
<developer>
- <id>shwethags</id>
- <name>Shwetha GS</name>
- <email>shwethags@apache.org</email>
- <timezone>Asia/Kolkata</timezone>
+ <id>apoorvnaik</id>
+ <name>Apoorv Naik</name>
+ <email>apoorvnaik@apache.org</email>
+ <timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
<!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>jmaron</id>
- <name>Jon Maron</name>
- <email>jmaron@hortonworks.com</email>
- <timezone>America/New_York</timezone>
+ <id>acmurthy</id>
+ <name>Arun C.Murthy</name>
+ <email>acmurthy@apache.org</email>
+ <timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
<!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>AAhn</id>
- <name>Andrew Ahn</name>
- <email>aahn@hortonworks.com</email>
- <timezone>America/Los_Angeles</timezone>
+ <id>ayubpathan</id>
+ <name>Ayub Pathan</name>
+ <email>ayubpathan@apache.org</email>
+ <timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
<!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>dkaspar</id>
- <name>David Kaspar</name>
- <email>dkaspar@merck.com</email>
- <timezone>Europe/Berlin</timezone>
- <roles>
- <role>committer</role>
- <role>PPMC</role>
- </roles>
- <organization>Merck & Co Inc.</organization>
- <!--<organizationUrl>http://www.merck.com</organizationUrl>-->
- </developer>
- <developer>
- <id>ilasek</id>
- <name>Ivo Lasek</name>
- <email>ilasek@merck.com</email>
- <timezone>Europe/Berlin</timezone>
- <roles>
- <role>committer</role>
- <role>PPMC</role>
- </roles>
- <organization>Merck & Co Inc.</organization>
- <!--<organizationUrl>http://www.merck.com</organizationUrl>-->
- </developer>
- <developer>
- <id>dfusaro</id>
- <name>Dennis Fusaro</name>
- <email>dfusaro@aetna.com</email>
- <timezone>America/New_York</timezone>
+ <id>bstortz</id>
+ <name>Barbara Stortz</name>
+ <email>bstortz@sap.com</email>
+ <timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Aetna Inc.</organization>
- <!--<organizationUrl>http://www.aetna.com</organizationUrl>-->
+ <organization>SAP Inc.</organization>
+ <!--<organizationUrl>http://www.sap.com</organizationUrl>-->
</developer>
<developer>
<id>chyzer</id>
@@ -190,7 +172,7 @@
<timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Aetna Inc.</organization>
<!--<organizationUrl>http://www.aetna.com</organizationUrl>-->
@@ -202,154 +184,151 @@
<timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Aetna Inc.</organization>
<!--<organizationUrl>http://www.aetna.com</organizationUrl>-->
</developer>
<developer>
- <id>gsenia</id>
- <name>Greg Senia</name>
- <email>gsenia@aetna.com</email>
- <timezone>America/New_York</timezone>
+ <id>darshankumar</id>
+ <name>Darshan Kumar</name>
+ <email>darshankumar@apache.org</email>
+ <timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Aetna Inc.</organization>
- <!--<organizationUrl>http://www.aetna.com</organizationUrl>-->
+ <organization>MPR</organization>
</developer>
<developer>
- <id>james</id>
- <name>James Vollmer</name>
- <email>james@target.com</email>
- <timezone>America/Denver</timezone>
+ <id>dkantor</id>
+ <name>Dave Kantor</name>
+ <email>dkantor@apache.org</email>
+ <timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Target Inc.</organization>
- <!--<organizationUrl>http://www.target.com</organizationUrl>-->
+ <organization>IBM Corporation</organization>
+ <!--<organizationUrl>http://www.ibm.com</organizationUrl>-->
</developer>
<developer>
- <id>adossett</id>
- <name>Aaron Dossett</name>
- <email>adossett@target.com</email>
- <timezone>America/Denver</timezone>
+ <id>dkaspar</id>
+ <name>David Kaspar</name>
+ <email>dkaspar@merck.com</email>
+ <timezone>Europe/Berlin</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Target Inc.</organization>
- <!--<organizationUrl>http://www.target.com</organizationUrl>-->
+ <organization>Merck & Co Inc.</organization>
+ <!--<organizationUrl>http://www.merck.com</organizationUrl>-->
</developer>
<developer>
- <id>mschussler</id>
- <name>Mitch Schussler</name>
- <email>mschussler@apache.org</email>
+ <id>dfusaro</id>
+ <name>Dennis Fusaro</name>
+ <email>dfusaro@aetna.com</email>
<timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>JPMC</organization>
- <!--<organizationUrl>http://</organizationUrl>-->
+ <organization>Aetna Inc.</organization>
+ <!--<organizationUrl>http://www.aetna.com</organizationUrl>-->
</developer>
<developer>
- <id>vavasarala</id>
- <name>Viswanath Avasarala</name>
- <email>vavasarala@slb.com</email>
- <timezone>America/Denver</timezone>
+ <id>gsenia</id>
+ <name>Greg Senia</name>
+ <email>gsenia@aetna.com</email>
+ <timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Schlumberger Inc.</organization>
- <!--<organizationUrl>http://www.slb.com</organizationUrl>-->
+ <organization>Aetna Inc.</organization>
+ <!--<organizationUrl>http://www.aetna.com</organizationUrl>-->
</developer>
<developer>
- <id>avarma</id>
- <name>Anil Varma</name>
- <email>avarma@slb.com</email>
- <timezone>America/Denver</timezone>
+ <id>rhbutani</id>
+ <name>Harish Butani</name>
+ <email>rhbutani@apache.org</email>
+ <timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Schlumberger Inc.</organization>
- <!--<organizationUrl>http://www.slb.com</organizationUrl>-->
+ <organization />
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>bstortz</id>
- <name>Barbara Stortz</name>
- <email>bstortz@sap.com</email>
- <timezone>America/Los_Angeles</timezone>
+ <id>yhemanth</id>
+ <name>Hemanth Yamijala</name>
+ <email>yhemanth@apache.org</email>
+ <timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>SAP Inc.</organization>
- <!--<organizationUrl>http://www.sap.com</organizationUrl>-->
+ <organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>sriksun</id>
- <name>Srikanth Sundarrajan</name>
- <email>sriksun@apache.org</email>
- <timezone>Asia/Kolkata</timezone>
+ <id>ilasek</id>
+ <name>Ivo Lasek</name>
+ <email>ilasek@merck.com</email>
+ <timezone>Europe/Berlin</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>InMobi</organization>
- <!--<organizationUrl>http://www.inmobi.com</organizationUrl>-->
+ <organization>Merck & Co Inc.</organization>
+ <!--<organizationUrl>http://www.merck.com</organizationUrl>-->
</developer>
<developer>
- <id>ssuresh</id>
- <name>Suresh Srinivas</name>
- <email>ssuresh@apache.org</email>
- <timezone>America/Los_Angeles</timezone>
+ <id>jnhagelberg</id>
+ <name>Jeffrey Hagelberg</name>
+ <email>jnhagelberg@apache.org</email>
+ <timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Hortonworks Inc.</organization>
- <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
+ <organization>IBM Corporation</organization>
+ <!--<organizationUrl>http://www.ibm.com</organizationUrl>-->
</developer>
<developer>
- <id>vranganathan</id>
- <name>Venkat Ranganathan</name>
- <email>venkatrangan@apache.org</email>
+ <id>jitendra</id>
+ <name>Jitendra Pandey</name>
+ <email>jitendra@apache.org</email>
<timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
<!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>sumasai</id>
- <name>Suma Shivaprasad</name>
- <email>sumasai@apache.org</email>
- <timezone>America/Los_Angeles</timezone>
+ <id>jmaron</id>
+ <name>Jon Maron</name>
+ <email>jmaron@apache.org</email>
+ <timezone>America/New_York</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Hortonworks Inc.</organization>
- <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
+ <organization></organization>
</developer>
<developer>
- <id>yhemanth</id>
- <name>Hemanth Yamijala</name>
- <email>yhemanth@apache.org</email>
+ <id>kalyanikashikar</id>
+ <name>Kalyani Kashikar</name>
+ <email>kalyanikashikar@apache.org</email>
<timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
- <organization>Hortonworks Inc.</organization>
- <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
+ <organization>Freestone Infotech</organization>
</developer>
<developer>
<id>kbhatt</id>
@@ -358,121 +337,162 @@
<timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
<organization>Freestone Infotech</organization>
</developer>
<developer>
- <id>dkantor</id>
- <name>Dave Kantor</name>
- <email>dkantor@apache.org</email>
- <timezone>America/New_York</timezone>
+ <id>madhan</id>
+ <name>Madhan Neethiraj</name>
+ <email>madhan@apache.org</email>
+ <timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization>IBM Corporation</organization>
- <!--<organizationUrl>http://www.ibm.com</organizationUrl>-->
+ <organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>darshankumar</id>
- <name>Darshan Kumar</name>
- <email>darshankumar@apache.org</email>
- <timezone>Asia/Kolkata</timezone>
+ <id>mschussler</id>
+ <name>Mitch Schussler</name>
+ <email>mschussler@apache.org</email>
+ <timezone>America/New_York</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization>MPR</organization>
+ <organization>JPMC</organization>
+ <!--<organizationUrl>http://</organizationUrl>-->
</developer>
<developer>
- <id>tbeerbower</id>
- <name>Tom Beerbower</name>
- <email>tbeerbower@apache.org</email>
+ <id>guptaneeru</id>
+ <name>Neeru Gupta</name>
+ <email>guptaneeru@us.ibm.com</email>
<timezone>America/New_York</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization></organization>
+ <organization>IBM Corporation</organization>
+ <!--<organizationUrl>http://www.ibm.com</organizationUrl>-->
</developer>
<developer>
- <id>madhan</id>
- <name>Madhan Neethiraj</name>
- <email>madhan@apache.org</email>
+ <id>nixon</id>
+ <name>Nixon Rodrigues</name>
+ <email>nixon@apache.org</email>
+ <timezone>Asia/Kolkata</timezone>
+ <roles>
+ <role>committer</role>
+ <role>PMC</role>
+ </roles>
+ <organization>Freestone Infotech</organization>
+ </developer>
+ <developer>
+ <id>sarath</id>
+ <name>Sarath Subramanian</name>
+ <email>sarath@apache.org</email>
<timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
- <role>PPMC</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>svimal2106</id>
- <name>Vimal Sharma</name>
- <email>svimal2106@apache.org</email>
+ <id>shwethags</id>
+ <name>Shwetha GS</name>
+ <email>shwethags@apache.org</email>
<timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization></organization>
+ <organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>jnhagelberg</id>
- <name>Jeffrey Hagelberg</name>
- <email>jnhagelberg@apache.org</email>
- <timezone>America/New_York</timezone>
+ <id>sriksun</id>
+ <name>Srikanth Sundarrajan</name>
+ <email>sriksun@apache.org</email>
+ <timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization>IBM Corporation</organization>
- <!--<organizationUrl>http://www.ibm.com</organizationUrl>-->
+ <organization>InMobi</organization>
+ <!--<organizationUrl>http://www.inmobi.com</organizationUrl>-->
</developer>
<developer>
- <id>apoorvnaik</id>
- <name>Apoorv Naik</name>
- <email>apoorvnaik@apache.org</email>
+ <id>sumasai</id>
+ <name>Suma Shivaprasad</name>
+ <email>sumasai@apache.org</email>
<timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>sarath</id>
- <name>Sarath Subramanian</name>
- <email>sarath@apache.org</email>
+ <id>ssuresh</id>
+ <name>Suresh Srinivas</name>
+ <email>ssuresh@apache.org</email>
<timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>ayubpathan</id>
- <name>Ayub Pathan</name>
- <email>ayubpathan@apache.org</email>
- <timezone>Asia/Kolkata</timezone>
+ <id>tbeerbower</id>
+ <name>Tom Beerbower</name>
+ <email>tbeerbower@apache.org</email>
+ <timezone>America/New_York</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
+ </roles>
+ <organization></organization>
+ </developer>
+ <developer>
+ <id>vranganathan</id>
+ <name>Venkat Ranganathan</name>
+ <email>venkatrangan@apache.org</email>
+ <timezone>America/Los_Angeles</timezone>
+ <roles>
+ <role>committer</role>
+ <role>PMC</role>
</roles>
<organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
<developer>
- <id>nixon</id>
- <name>Nixon Rodrigues</name>
- <email>nixon@apache.org</email>
- <timezone>Asia/Kolkata</timezone>
+ <id>venkatesh</id>
+ <name>Venkatesh Seetharam</name>
+ <email>venkatesh@apache.org</email>
+ <timezone>America/Los_Angeles</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization>Freestone Infotech</organization>
+ <organization></organization>
</developer>
<developer>
- <id>kalyanikashikar</id>
- <name>Kalyani Kashikar</name>
- <email>kalyanikashikar@apache.org</email>
+ <id>svimal2106</id>
+ <name>Vimal Sharma</name>
+ <email>svimal2106@apache.org</email>
<timezone>Asia/Kolkata</timezone>
<roles>
<role>committer</role>
+ <role>PMC</role>
</roles>
- <organization>Freestone Infotech</organization>
+ <organization>Hortonworks Inc.</organization>
+ <!--<organizationUrl>http://www.hortonworks.com</organizationUrl>-->
</developer>
</developers>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/repository/pom.xml
----------------------------------------------------------------------
diff --git a/repository/pom.xml b/repository/pom.xml
index 2573198..8e8ccab 100755
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-repository</artifactId>
<description>Apache Atlas Repository Module</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/server-api/pom.xml
----------------------------------------------------------------------
diff --git a/server-api/pom.xml b/server-api/pom.xml
index 7761a71..a26e728 100644
--- a/server-api/pom.xml
+++ b/server-api/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/shaded/hbase-client-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/shaded/hbase-client-shaded/pom.xml b/shaded/hbase-client-shaded/pom.xml
index fa1f290..fe59680 100644
--- a/shaded/hbase-client-shaded/pom.xml
+++ b/shaded/hbase-client-shaded/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>atlas-hbase-client-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/shaded/hbase-server-shaded/pom.xml
----------------------------------------------------------------------
diff --git a/shaded/hbase-server-shaded/pom.xml b/shaded/hbase-server-shaded/pom.xml
index 719990e..fd61ac9 100644
--- a/shaded/hbase-server-shaded/pom.xml
+++ b/shaded/hbase-server-shaded/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>apache-atlas</artifactId>
<groupId>org.apache.atlas</groupId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>atlas-hbase-server-shaded</artifactId>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/typesystem/pom.xml
----------------------------------------------------------------------
diff --git a/typesystem/pom.xml b/typesystem/pom.xml
index 1b7bf32..73d9fba 100755
--- a/typesystem/pom.xml
+++ b/typesystem/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-typesystem</artifactId>
<description>Apache Atlas Typesystem Module</description>
http://git-wip-us.apache.org/repos/asf/atlas/blob/de251913/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 465d2a5..62bba3f 100755
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
- <version>0.9-incubating-SNAPSHOT</version>
+ <version>0.9-SNAPSHOT</version>
</parent>
<artifactId>atlas-webapp</artifactId>
<description>Apache Atlas Web Application</description>
@@ -540,7 +540,7 @@
</execution>
</executions>
<configuration>
- <dname>cn=atlas.incubator.apache.org</dname>
+ <dname>cn=atlas.apache.org</dname>
<keystore>${project.build.directory}/atlas.keystore</keystore>
<keypass>keypass</keypass>
<storepass>keypass</storepass>