You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pe...@apache.org on 2016/08/22 04:47:55 UTC
[02/16] falcon git commit: FALCON-2118 Proposal for new UI changes
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/js/services/entity/entity-factory.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/entity/entity-factory.js b/falcon-ui/app/js/services/entity/entity-factory.js
index 82848c0..c8f1707 100644
--- a/falcon-ui/app/js/services/entity/entity-factory.js
+++ b/falcon-ui/app/js/services/entity/entity-factory.js
@@ -51,14 +51,22 @@
return new Location(type, path);
},
- newCluster: function (type, selected) {
- return new Cluster(type, selected);
+ newCluster: function (type, selected, name, partition) {
+ return new Cluster(type, selected, name, partition);
+ },
+
+ newPartition: function (name) {
+ return new Partition(name);
},
newEntry: function (key, value) {
return new Entry(key, value);
},
+ newProperty: function (name, value) {
+ return new Property(name, value);
+ },
+
newProcess: function () {
return new Process();
},
@@ -71,12 +79,53 @@
return new Output();
},
+ newClusterEntity: function() {
+ return new ClusterEntity();
+ },
+
+ newClusterLocation: function(name, path) {
+ return new ClusterLocation(name, path);
+ },
+
+ newClusterInterface: function(type, endpoint, version) {
+ return new ClusterInterface(type, endpoint, version);
+ },
+
+ newSnapshot: function() {
+ return new Snapshot();
+ },
+
+ newSparkAttributes: function() {
+ return new SparkAttributes();
+ },
+
+ newCredential: function() {
+ return new Credential();
+ },
+
+ newDatasourceInterface: function() {
+ return new DatasourceInterface();
+ },
+
+ newDatasource: function() {
+ return new Datasource();
+ },
+
+ newClusterFileSystem: function() {
+ return new clusterFileSystem();
+ },
+
newEntity: function (type) {
if (type === 'feed') {
return this.newFeed();
- }
- if (type === 'process') {
+ } else if (type === 'process') {
return this.newProcess();
+ } else if (type === 'cluster') {
+ return this.newClusterEntity();
+ } else if (type === 'snapshot') {
+ return this.newSnapshot();
+ } else if (type === 'datasource') {
+ return this.newDatasource();
}
}
@@ -91,14 +140,19 @@
this.tags = [new Entry(null, null)];
this.ACL = new ACL();
this.schema = new Schema();
- this.frequency = new Frequency(null, 'hours');
+ this.frequency = new Frequency(1, 'hours');
this.lateArrival = new LateArrival();
- this.availabilityFlag = null;
+ this.availabilityFlag = '_success';
this.properties = feedProperties();
this.customProperties = [new Entry(null, null)];
this.storage = new Storage();
- this.clusters = [new Cluster('source', true)];
- this.timezone = "";
+ this.clusters = [];
+ this.timezone = 'UTC';
+ this.partitions = [];
+ this.retentionFrequency = new Frequency(20, 'minutes');
+ this.validity = new Validity();
+ this.enableFeedReplication = false;
+ this.dataTransferType = '';
}
@@ -109,15 +163,14 @@
}
function Schema() {
- this.location = undefined;
- this.provider = undefined;
+ this.location = '/none';
+ this.provider = '/none';
}
function feedProperties() {
return [
- new Entry('queueName', ''),
- new Entry('jobPriority', ''),
- new Entry('timeout', ''),
+ new Entry('queueName', 'default'),
+ new Entry('jobPriority', 'NORMAL'),
new Entry('parallel', ''),
new Entry('maxMaps', ''),
new Entry('mapBandwidthKB', '')
@@ -138,8 +191,8 @@
}
function LateArrival() {
- this.active = false;
- this.cutOff = new Frequency(null, 'hours');
+ this.active = true;
+ this.cutOff = new Frequency(4, 'hours');
}
function Frequency(quantity, unit) {
@@ -156,13 +209,15 @@
this.fileSystem = new FileSystem();
this.catalog = new Catalog();
}
- function clusterStorage() {
- this.fileSystem = new clusterFileSystem();
- this.catalog = new Catalog();
+ function clusterStorage(type) {
+ if(type === 'hdfs'){
+ this.fileSystem = new clusterFileSystem();
+ }else if(type === 'hive'){
+ this.catalog = new Catalog();
+ }
}
function Catalog() {
- this.active = false;
this.catalogTable = new CatalogTable();
}
@@ -172,12 +227,10 @@
}
function FileSystem() {
- this.active = true;
- this.locations = [new Location('data','/'), new Location('stats','/'), new Location('meta','/')];
+ this.locations = null;
}
function clusterFileSystem() {
- this.active = false;
- this.locations = [ new Location('data',''), new Location('stats',''), new Location('meta','') ];
+ this.locations = [ new Location('data',''), new Location('stats','/')];
}
function Location(type, path) {
@@ -186,20 +239,36 @@
this.focused = false;
}
- function Cluster(type, selected) {
+ function Cluster(type, selected, name, partition) {
// this.name = null;
- this.name = "";
+ this.name = (name != undefined) ? name : "";
this.type = type;
this.selected = selected;
- this.retention = new Frequency(null, 'hours');
+ if (type == 'source') {
+ this.retention = new Frequency(90, 'days');
+ } else if (type == 'target') {
+ this.retention = new Frequency(12, 'months');
+ } else {
+ this.retention = new Frequency(null, 'hours');
+ }
+
this.retention.action = 'delete';
this.validity = new Validity();
- this.storage = new clusterStorage();
+ this.storage = new clusterStorage(selected);
+ if (partition != undefined) {
+ this.partition = partition;
+ }
+ }
+
+ function Partition(name) {
+ this.name = name;
}
function Validity() {
this.start = new DateAndTime();
this.end = new DateAndTime();
+ this.end.date = new Date("Dec 31, 2099 11:59:59");
+ this.end.time = new Date("Dec 31, 2099 11:59:59");
this.timezone = "";
}
@@ -222,15 +291,17 @@
this.name = null;
this.tags = [new Entry(null, null)];
this.workflow = new Workflow();
- this.timezone = "";
- this.frequency = new Frequency(null, 'hours');
+ this.timezone = 'UTC';
+ this.frequency = new Frequency(30, 'minutes');
this.parallel = 1;
- this.order = "";
- this.retry = new Retry();
+ this.order = 'FIFO';
+ this.retry = new ProcessRetry();
this.clusters = [new Cluster('source', true)];
this.inputs = [];
this.outputs = [];
this.ACL = new ACL();
+ this.properties = [new Property(null, null)];
+ this.validity = new Validity();
/*
this.name = 'P';
@@ -245,17 +316,34 @@
*/
}
+ function SparkAttributes() {
+ this.name = '';
+ this.master = 'yarn';
+ this.mode = 'cluster';
+ this.class = '';
+ this.sparkOptions = '';
+ this.jar = '';
+ this.arg = '';
+ }
+
function Workflow() {
- this.name = "";
- this.engine = "";
+ this.name = '';
+ this.engine = '';
this.version = '';
this.path = '/';
+ this.spark = new SparkAttributes();
}
function Retry() {
- this.policy = '';
- this.attempts = null;
- this.delay = new Frequency(null, '');
+ this.policy = 'exp-backoff';
+ this.attempts = 3;
+ this.delay = new Frequency(30, 'minutes');
+ }
+
+ function ProcessRetry() {
+ this.policy = 'periodic';
+ this.attempts = 3;
+ this.delay = new Frequency(30, 'minutes');
}
function Input() {
@@ -268,7 +356,115 @@
function Output() {
this.name = null;
this.feed = "";
- this.outputInstance = null;
+ this.outputInstance = 'now(0,0)';
}
-})();
\ No newline at end of file
+ function ClusterEntity() {
+ this.name = "";
+ this.colo = null;
+ this.description = null;
+ this.tags = [new Entry(null, null)];
+ this.ACL = new ACL();
+
+ this.interfaces = [];
+ this.properties = [];
+ this.locations = []
+ }
+
+ function ClusterLocation(name, path) {
+ this.name = name;
+ this.path= path;
+ }
+
+ function ClusterInterface(type, endpoint, version) {
+ this.type = type;
+ this.endpoint = endpoint;
+ this.version = version;
+ }
+
+ function SnapshotCluster(type) {
+ this.cluster = '';
+ this.directoryPath = '';
+ if (type === 'source') {
+ this.deleteFrequency = new Frequency(14, 'days');
+ this.retentionNumber = 90;
+ } else if (type === 'target') {
+ this.deleteFrequency = new Frequency(14, 'days');
+ this.retentionNumber = 90;
+ }
+ }
+
+ function Snapshot() {
+ this.name = '';
+ this.type = 'snapshot';
+ this.ACL = new ACL();
+ this.tags = [new Entry(null, null)];
+ this.frequency = new Frequency(1, 'days');
+ this.alerts = [];
+ this.validity = new Validity();
+ this.validity.timezone = 'UTC';
+ this.runOn = 'target';
+ this.retry = new Retry();
+ this.source = new SnapshotCluster('source');
+ this.target = new SnapshotCluster('target');
+ this.allocation = {};
+ this.tdeEncryptionEnabled = false;
+ }
+
+ function Credential() {
+ this.type = "";
+ this.userName = ""
+ this.passwordText = "";
+ this.passwordFile = "";
+ this.passwordAlias = "";
+ this.providerPath = "";
+ }
+
+ function DatasourceInterface() {
+ this.type = "readonly";
+ this.endpoint = "";
+ this.credential = new Credential();
+ }
+
+ function DatasourceInterfaces() {
+ this.credential = new Credential();
+ this.interfaces = [new DatasourceInterface()];
+ }
+
+ function Driver() {
+ this.clazz = null;
+ this.jar = [{value:""}];
+ }
+
+ function Property(name, value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ function datasourceProperties() {
+ return [
+ new Property('parameterFile', ''),
+ new Property('verboseMode', ''),
+ new Property('directMode', '')
+ ];
+ }
+
+ function Datasource() {
+ this.name = "";
+ this.colo = null;
+ this.description = null;
+ this.tags = [new Entry(null, null)];
+ this.type = "";
+ this.customProperties = [];
+ this.properties = new datasourceProperties();
+ this.parameters = [];
+ this.ACL = new ACL();
+ this.interfaces = new DatasourceInterfaces();
+ this.host = "";
+ this.port = "";
+ this.databaseName = "";
+ this.driver = new Driver();
+ }
+
+
+})();
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/js/services/entity/entity-model.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/entity/entity-model.js b/falcon-ui/app/js/services/entity/entity-model.js
index 48fa617..a8a66d7 100644
--- a/falcon-ui/app/js/services/entity/entity-model.js
+++ b/falcon-ui/app/js/services/entity/entity-model.js
@@ -21,15 +21,20 @@
module.factory('EntityModel', ["X2jsService", "$cookieStore", function(X2jsService, $cookieStore) {
- var EntityModel = {}, userName;
+ var EntityModel = {};
EntityModel.json = null;
EntityModel.detailsPageModel = null;
+ EntityModel.getUserNameFromCookie = function() {
+ return $cookieStore.get('userToken')?$cookieStore.get('userToken').user:"";
+ };
+
EntityModel.identifyType = function(json) {
if(json && json.feed) { EntityModel.type = "feed"; }
else if(json && json.cluster) { EntityModel.type = "cluster"; }
else if(json && json.process) { EntityModel.type = "process"; }
+ else if(json && json.datasource) { EntityModel.type = "datasource"; }
else { EntityModel.type = 'Type not recognized'; }
};
@@ -38,11 +43,6 @@
return EntityModel.identifyType(EntityModel.json);
};
- if($cookieStore.get('userToken')){
- userName = $cookieStore.get('userToken').user;
- } else {
- userName = "";
- }
EntityModel.defaultValues = {
cluster: {
@@ -52,45 +52,50 @@
interface:[
{
_type:"readonly",
- _endpoint:"hftp://sandbox.hortonworks.com:50070",
+ _endpoint:"hftp://<hostname>:50070",
_version:"2.2.0"
},
{
_type:"write",
- _endpoint:"hdfs://sandbox.hortonworks.com:8020",
+ _endpoint:"hdfs://<hostname>:8020",
_version:"2.2.0"
},
{
_type:"execute",
- _endpoint:"sandbox.hortonworks.com:8050",
+ _endpoint:"<hostname>:8050",
_version:"2.2.0"
},
{
_type:"workflow",
- _endpoint:"http://sandbox.hortonworks.com:11000/oozie/",
+ _endpoint:"http://<hostname>:11000/oozie/",
_version:"4.0.0"
},
{
_type:"messaging",
- _endpoint:"tcp://sandbox.hortonworks.com:61616?daemon=true",
+ _endpoint:"tcp://<hostname>:61616?daemon=true",
_version:"5.1.6"
+ },
+ {
+ _type:"registry",
+ _endpoint:"thrift://<hostname>:9083",
+ _version:"0.11.0"
+
}
]
},
locations:{
location:[
{_name: "staging", _path: ""},
- {_name: "temp", _path: ""},
- {_name: "working", _path: ""},
- {"_name":"","_path":""} //>> to compare
+ {_name: "temp", _path: "/tmp"},
+ {_name: "working", _path: ""}
]
},
ACL: {
- _owner: userName,
+ _owner: EntityModel.getUserNameFromCookie(),
_group: "users",
_permission: "0x755"
},
@@ -107,13 +112,10 @@
},
MirrorUIModel: {
name: undefined,
- tags: {
- newTag: { value:"", key:"" },
- tagsArray: [{ key:"_falcon_mirroring_type", value:"HDFS" }],
- tagsString: ""
- },
- formType: "HDFS",
+ tags: [{ value:"", key:"" }],
+ type: "HDFS",
runOn: "target",
+ tdeEncryptionEnabled: true,
source: {
location: "HDFS",
cluster: "",
@@ -121,42 +123,43 @@
path: "",
hiveDatabaseType: "databases",
hiveDatabases: "",
- hiveDatabase: "",
- hiveTables: ""
+ hiveTables: "",
+ hiveMetastoreUri : "thrift://localhost:9083",
+ hive2KerberosPrincipal : "hive/_HOST@EXAMPLE.COM",
+ hiveMetastoreKerberosPrincipal : "hive/_HOST@EXAMPLE.COM"
},
target: {
location: "HDFS",
cluster: "",
url: "",
- path: ""
- },
- alerts: {
- alert: { email: "" },
- alertsArray: []
+ path: "",
+ hive2KerberosPrincipal : "hive/_HOST@EXAMPLE.COM",
+ hiveMetastoreUri : "thrift://localhost:9083",
+ hiveMetastoreKerberosPrincipal : "hive/_HOST@EXAMPLE.COM"
},
+ alerts: [],
validity: {
- start: (function () { var d = new Date(); d.setHours(0); d.setMinutes(0); d.setSeconds(0); return d; }()),
- startTime: new Date(),
- end: "",
- endTime: new Date(),
- tz: "GMT+00:00",
+ start: {date: (function () { var d = new Date(); d.setHours(0); d.setMinutes(0); d.setSeconds(0); return d; }()),
+ time: new Date()},
+ end: {date: new Date("Dec 31, 2099 11:59:59"), time: new Date("Dec 31, 2099 11:59:59")},
+ timezone: "UTC",
startISO: "",
endISO: ""
},
frequency: {
- number: 5,
- unit: 'minutes'
+ quantity: 1,
+ unit: 'days'
},
allocation: {
hdfs:{
- maxMaps: "5",
- maxBandwidth: "100"
+ distcpMaxMaps: "5",
+ distcpMapBandwidth: "100"
},
hive:{
- maxMapsDistcp: "1",
- maxMapsMirror: "5",
- maxMapsEvents: "-1",
- maxBandwidth: "100"
+ distcpMaxMaps: "1",
+ replicationMaxMaps: "5",
+ maxEvents: "-1",
+ distcpMapBandwidth: "100"
}
},
hiveOptions: {
@@ -173,14 +176,14 @@
policy:"periodic",
delay: {
unit: "minutes",
- number: 30
+ quantity: 30
},
attempts: 3
},
- acl: {
- owner: userName,
+ ACL: {
+ owner: EntityModel.getUserNameFromCookie(),
group: "users",
- permissions: "0x755"
+ permission: "0x755"
}
}
};
@@ -191,7 +194,7 @@
tags: "",
groups: "",
frequency: "",
- /*timezone: "GMT+00:00",*/
+ /*timezone: "UTC",*/
"late-arrival": {
"_cut-off": ""
},
@@ -222,7 +225,7 @@
}]
},
ACL: {
- _owner: userName,
+ _owner: EntityModel.getUserNameFromCookie(),
_group: "users",
_permission: "0x755"
},
@@ -241,7 +244,7 @@
UIModel: {},
HDFS: {
process: {
- tags: "",
+ tags: [{ value:"", key:"" }],
clusters: {
cluster: [{
validity: {
@@ -253,7 +256,7 @@
},
parallel: "1",
order: "LAST_ONLY",
- frequency: "minutes(5)",
+ frequency: "days(1)",
timezone: "UTC",
properties: {
property: [
@@ -296,13 +299,16 @@
{
_name: "sourceCluster",
_value: ""
+ }, {
+ _name: "tdeEncryptionEnabled",
+ _value: "true"
}
]
},
workflow: {
_name: "hdfs-dr-workflow",
_engine: "oozie",
- _path: "/apps/data-mirroring/workflows/hdfs-replication-workflow.xml",
+ _path: "/apps/falcon/extensions/hdfs-mirroring/resources/runtime/hdfs-mirroring-workflow.xml",
_lib: ""
},
retry: {
@@ -321,7 +327,7 @@
},
HIVE: {
process: {
- tags: "",
+ tags: [{ value:"", key:"" }],
clusters: {
cluster: [{
validity: {
@@ -333,7 +339,7 @@
},
parallel: "1",
order: "LAST_ONLY",
- frequency: "minutes(3)",
+ frequency: "days(1)",
timezone: "UTC",
properties: {
property: [
@@ -391,11 +397,27 @@
},
{
_name: "targetMetastoreUri",
- _value: "thrift://240.0.0.11:9083"
+ _value: "thrift://localhost:9083"
},
{
_name: "sourceMetastoreUri",
- _value: "thrift://240.0.0.10:9083"
+ _value: "thrift://localhost:9083"
+ },
+ {
+ _name: "targetHiveMetastoreKerberosPrincipal",
+ _value: "hive/_HOST@EXAMPLE.COM"
+ },
+ {
+ _name: "sourceHiveMetastoreKerberosPrincipal",
+ _value: "hive/_HOST@EXAMPLE.COM"
+ },
+ {
+ _name: "targetHive2KerberosPrincipal",
+ _value: "hive/_HOST@EXAMPLE.COM"
+ },
+ {
+ _name: "sourceHive2KerberosPrincipal",
+ _value: "hive/_HOST@EXAMPLE.COM"
},
{
_name: "sourceTable",
@@ -428,13 +450,16 @@
{
_name: "drNotificationReceivers",
_value: "NA"
+ }, {
+ _name: "tdeEncryptionEnabled",
+ _value: "true"
}
]
},
workflow: {
_name: "falcon-dr-hive-workflow",
_engine: "oozie",
- _path: "/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml",
+ _path: "/apps/falcon/extensions/hive-mirroring/resources/runtime/hive-mirroring-workflow.xml",
_lib: ""
},
retry: {
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/js/services/entity/entity-scheduler.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/entity/entity-scheduler.js b/falcon-ui/app/js/services/entity/entity-scheduler.js
new file mode 100644
index 0000000..cbc544d
--- /dev/null
+++ b/falcon-ui/app/js/services/entity/entity-scheduler.js
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ 'use strict';
+
+ var entityScheduler = angular.module("app.services.entity.scheduler",['app.services.falcon']);
+ entityScheduler.factory('EntityScheduler',["$window", "$q", "Falcon", "EncodeService", "X2jsService", function($window, $q, Falcon, encodeService, X2jsService){
+ var entityScheduler = {};
+ var entityStatus = {
+ RUNNING : "RUNNING",
+ SUBMITTED : "SUBMITTED",
+ SUSPENDED : "SUSPENDED",
+ DELETED : "DELETED",
+ FAILED : "FAILED"
+ };
+
+ entityScheduler.resumeEntity = function (type, name) {
+ var deferred = $q.defer();
+ type = type.toLowerCase();
+ Falcon.logRequest();
+ Falcon.postResumeEntity(type, name).success(function (data) {
+ Falcon.logResponse('success', data, type);
+ deferred.resolve(entityStatus.RUNNING);
+ })
+ .error(function (err) {
+ Falcon.logResponse('error', err, type);
+ deferred.resolve(entityStatus.SUSPENDED);
+ });
+ return deferred.promise;
+ };
+
+ entityScheduler.scheduleEntity = function (type, name) {
+ var deferred = $q.defer();
+ type = type.toLowerCase();
+ Falcon.logRequest();
+ Falcon.postScheduleEntity(type, name).success(function (data) {
+ Falcon.logResponse('success', data, type);
+ deferred.resolve(entityStatus.RUNNING);
+ })
+ .error(function (err) {
+ Falcon.logResponse('error', err, type);
+ deferred.resolve(entityStatus.SUBMITTED);
+ });
+ return deferred.promise;
+ };
+
+ entityScheduler.suspendEntity = function (type, name) {
+ var deferred = $q.defer();
+ Falcon.logRequest();
+ type = type.toLowerCase();
+ Falcon.postSuspendEntity(type, name)
+ .success(function (message) {
+ Falcon.logResponse('success', message, type);
+ deferred.resolve(entityStatus.SUSPENDED);
+ })
+ .error(function (err) {
+ Falcon.logResponse('error', err, type);
+ deferred.resolve(entityStatus.RUNNING);
+ });
+ return deferred.promise;
+ };
+
+ entityScheduler.deleteEntity = function (type, name) {
+ type = type.toLowerCase(); //new sandbox returns uppercase type
+ var deferred = $q.defer();
+ Falcon.logRequest();
+ Falcon.deleteEntity(type, name)
+ .success(function (data) {
+ Falcon.logResponse('success', data, type);
+ deferred.resolve(entityStatus.DELETED);
+ })
+ .error(function (err) {
+ Falcon.logResponse('error', err, type);
+ deferred.resolve(entityStatus.FAILED);
+ });
+ return deferred.promise;
+ };
+
+ entityScheduler.getEntityDefinition = function(type, name){
+ var deferred = $q.defer();
+ type = type.toLowerCase(); //new sandbox returns uppercase type
+ Falcon.logRequest();
+ Falcon.getEntityDefinition(type, name)
+ .success(function (data) {
+ Falcon.logResponse('success', data, false, true);
+ deferred.resolve(X2jsService.xml_str2json(data));
+ })
+ .error(function (err) {
+ Falcon.logResponse('error', err, false, true);
+ deferred.reject(entityStatus.FAILED);
+ });
+ return deferred.promise;
+ };
+
+ entityScheduler.downloadEntity = function (type, name) {
+ type = type.toLowerCase();
+ Falcon.logRequest();
+ Falcon.getEntityDefinition(type, name) .success(function (data) {
+ Falcon.logResponse('success', data, false, true);
+ $window.location.href = 'data:application/octet-stream,' + encodeService.encode(data);
+ }).error(function (err) {
+ Falcon.logResponse('error', err, false);
+ });
+ };
+ return entityScheduler;
+ }]);
+})();
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/js/services/entity/entity-serializer.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/entity/entity-serializer.js b/falcon-ui/app/js/services/entity/entity-serializer.js
index 284c30f..327a311 100644
--- a/falcon-ui/app/js/services/entity/entity-serializer.js
+++ b/falcon-ui/app/js/services/entity/entity-serializer.js
@@ -38,8 +38,7 @@
return DateHelper.importDate(input, feedTz);
}
function timeAndDateToStringProcess(input) {
- //return DateHelper.createISO(input.date, input.time, processTz);
- return DateHelper.createISOString(input.date, input.time);
+ return DateHelper.createISO(input.date, input.time, processTz);
}
function importDateProcess (input) {
return DateHelper.importDate(input, processTz);
@@ -47,19 +46,24 @@
return {
- preSerialize: function(feed, type) {
+ preSerialize: function(entity, type) {
if(type === 'feed') {
- if(feed.properties) {
- feed.allproperties = feed.properties.concat(feed.customProperties);
+ if(entity.properties) {
+ entity.allproperties = entity.properties.concat(entity.customProperties);
}
- return preSerializeFeed(feed, JsonTransformerFactory);
+ return preSerializeFeed(entity, JsonTransformerFactory);
} else if(type === 'process') {
- return preSerializeProcess(feed, JsonTransformerFactory);
+ return preSerializeProcess(entity, JsonTransformerFactory);
+ } else if(type === 'datasource') {
+ if(entity.properties) {
+ entity.allProperties = entity.properties.concat(entity.customProperties);
+ }
+ return preSerializeDatasource(entity, JsonTransformerFactory);
}
},
- serialize: function(feed, type) {
- return X2jsService.json2xml_str(this.preSerialize(feed, type));
+ serialize: function(entity, type) {
+ return X2jsService.json2xml_str(this.preSerialize(entity, type));
},
preDeserialize: function(entityModel, type) {
@@ -67,6 +71,10 @@
return preDeserializeFeed(entityModel, JsonTransformerFactory);
} else if(type === 'process') {
return preDeserializeProcess(entityModel, JsonTransformerFactory);
+ } else if(type === 'cluster') {
+ return preDeserializeCluster(entityModel, JsonTransformerFactory);
+ } else if(type === 'datasource') {
+ return preDeserializeDatasource(entityModel, JsonTransformerFactory);
}
},
@@ -89,6 +97,11 @@
return input && input.value;
}
+ function emptyProperty (property) {
+ return property && property.name && property.name !== ''
+ && property.value && property.value !== '';
+ }
+
function emptyFrequency (input) {
return input.value.unit ? input.value.quantity : input.value;
}
@@ -124,39 +137,141 @@
.transform('type', '_type')
.transform('path', '_path');
+ var partitionTransform = transformerFactory
+ .transform('name', '_name');
+
var clusterTransform = transformerFactory
- .transform('name', '_name')
- .transform('type', '_type')
- .transform('validity.start', 'validity._start', (function () { feedTz = feed.timezone; return timeAndDateToStringFeed; }()))
- .transform('validity.end', 'validity._end', timeAndDateToStringFeed)
- .transform('retention', 'retention._limit', frequencyToString)
- .transform('retention.action', 'retention._action')
- .transform('storage.fileSystem', 'locations.location', function(fileSystem) {
- return feed.storage.fileSystem.active ? transformfileSystem(fileSystem) : null;
+ .transform('cluster.name', '_name')
+ .transform('cluster.type', '_type')
+ .transform('cluster.partition', '_partition')
+ .transform('cluster.validity.start', 'validity._start', (function () { feedTz = feed.timezone; return timeAndDateToStringFeed; }()))
+ .transform('cluster.validity.end', 'validity._end', timeAndDateToStringFeed)
+ .transform('cluster.retention', 'retention._limit', frequencyToString)
+ .transform('cluster.retention.action', 'retention._action')
+ .transform('cluster','import',function(cluster){
+ if(feed.dataTransferType === 'import' && feed.import){
+ return dataSourceTransformImport.apply(feed.import, {});
+ }
})
- .transform('storage.catalog', 'table', function(catalog) {
- return feed.storage.catalog.active ? transformCatalog(catalog) : null;
- });
+ .transform('cluster','export',function(cluster){
+ if(feed.dataTransferType === 'export' && feed.export){
+ return dataSourceTransformExport.apply(feed.export, {});
+ }
+ })
+ .transform('cluster', 'locations.location', function(cluster) {
+ if ((cluster.type === 'source' && feed.sourceClusterLocationType === 'hdfs')
+ || (cluster.type === 'target' && feed.targetClusterLocationType === 'hdfs')
+ || (feed.dataTransferType === 'import' && feed.targetClusterLocationType === 'hdfs')
+ || (feed.dataTransferType === 'export' && feed.sourceClusterLocationType === 'hdfs')
+ ) {
+ return transformfileSystem(cluster.storage.fileSystem);
+ }
+ return null;
+ })
+ .transform('cluster', 'table', function(cluster) {
+ if ((cluster.type === 'source' && feed.sourceClusterLocationType === 'hive')
+ || (cluster.type === 'target' && feed.targetClusterLocationType === 'hive')
+ || (feed.dataTransferType === 'import' && feed.targetClusterLocationType === 'hive')
+ || (feed.dataTransferType === 'export' && feed.sourceClusterLocationType === 'hive')
+ ) {
+ return transformCatalog(cluster.storage.catalog);
+ }
+ return null;
+ })
+ ;
+
+ var fieldTransform = transformerFactory
+ .transform('field', 'field');
+
+ var dataSourceTransformImport = transformerFactory
+ .transform('source.name','source._name')
+ .transform('source.tableName','source._tableName')
+ .transform('source.extract.type','source.extract._type')
+ .transform('source.extract.mergepolicy','source.extract.mergepolicy')
+ .transform('source','source.fields.includes.field',function(source){
+ if (source.columnsType === 'include' && source.fields.includes && source.fields.includes.length > 0) {
+ return source.fields.includes.map(function (field) {
+ return fieldTransform.apply(field, field);
+ });
+ }
+ return null;
+ })
+ .transform('source','source.fields.excludes.field',function(source){
+ if(source.columnsType === 'exclude' && source.fields.excludes && source.fields.excludes.length > 0){
+ return source.fields.excludes.map(function (field) {
+ return fieldTransform.apply(field, field);
+ });
+ }
+ return null;
+ })
+ ;
+
+ var dataSourceTransformExport = transformerFactory
+ .transform('target.name','target._name')
+ .transform('target.tableName','target._tableName')
+ .transform('target.load.type','target.load._type')
+ .transform('target','target.fields.includes.field',function(target){
+ if(target.columnsType === 'include' && target.fields.includes && target.fields.includes.length > 0){
+ return target.fields.includes.map(function (field) {
+ return fieldTransform.apply(field, field);
+ });
+ }
+ return null;
+ })
+ .transform('target','target.fields.excludes.field',function(target){
+ if(target.columnsType === 'exclude' && target.fields.excludes && target.fields.excludes.length > 0){
+ return target.fields.excludes.map(function (field) {
+ return fieldTransform.apply(field, field);
+ });
+ }
+ return null;
+ })
+ ;
var transform = transformerFactory
.transform('name', 'feed._name')
.transform('description', 'feed._description')
.transform('tags', 'feed.tags', keyValuePairs)
+ .transform('partitions', 'feed.partitions.partition', function(partitions) {
+ return partitions.length==0 ? null : partitions.map(function(partition) {
+ return partitionTransform.apply(partition, {});
+ });
+ })
.transform('groups', 'feed.groups')
.transform('availabilityFlag', 'feed.availabilityFlag')
.transform('frequency', 'feed.frequency', frequencyToString)
.transform('timezone', 'feed.timezone')
.transform('lateArrival.cutOff', 'feed.late-arrival._cut-off', frequencyToString)
.transform('clusters', 'feed.clusters.cluster', function(clusters) {
+ clusters = clusters.filter(function (cluster) {
+ return cluster.name;
+ });
+ if (!feed.enableFeedReplication) {
+ clusters = clusters.filter(function (cluster) {
+ return cluster.type == 'source';
+ });
+ }
return clusters.map(function(cluster) {
- return clusterTransform.apply(cluster, {});
+ return clusterTransform.apply({'cluster':cluster}, {});
});
})
.transform('storage.fileSystem', 'feed.locations.location', function(fileSystem) {
- return fileSystem.active ? transformfileSystem(fileSystem) : null;
+ if (feed.dataTransferType === 'hdfs'
+ || (feed.dataTransferType === 'import' && feed.targetClusterLocationType === 'hdfs')
+ || (feed.dataTransferType === 'export' && feed.sourceClusterLocationType === 'hdfs')
+ || (feed.dataTransferType === 'hive' && feed.sourceClusterLocationType === 'hdfs')) {
+ return transformfileSystem(fileSystem)
+ }
+ return null;
})
.transform('storage.catalog', 'feed.table', function(catalog) {
- return catalog.active ? transformCatalog(catalog) : null;
+ if (feed.dataTransferType === 'hive'
+ || (feed.dataTransferType === 'import' && feed.targetClusterLocationType === 'hive')
+ || (feed.dataTransferType === 'export' && feed.sourceClusterLocationType === 'hive')
+ || (feed.dataTransferType === 'hdfs' && feed.sourceClusterLocationType === 'hive')) {
+ return transformCatalog(catalog);
+ }
+ return null;
})
.transform('ACL', 'feed.ACL', emptyElement)
.transform('ACL.owner', 'feed.ACL._owner')
@@ -166,10 +281,13 @@
.transform('schema.location', 'feed.schema._location')
.transform('schema.provider', 'feed.schema._provider')
.transform('allproperties', 'feed.properties.property', function(properties) {
- return properties.filter(emptyValue).filter(emptyFrequency).map(function(property) {
+ properties = properties.filter(emptyValue);
+ return properties.length==0 ? null : properties.map(function(property) {
return propertyTransform.apply(property, {});
});
- });
+ })
+ //.transform('retentionFrequency', 'feed.lifecycle.retention-stage.frequency', frequencyToString)
+ ;
function transformfileSystem (fileSystem) {
return fileSystem.locations.map(function(location) {
@@ -203,12 +321,42 @@
.transform('feed', '_feed')
.transform('outputInstance', '_instance');
+ var propertyTransform = transformerFactory
+ .transform('name', '_name')
+ .transform('value', '_value');
+
+ var sparkTransform = transformerFactory
+ .transform('spark', 'master', function(spark) {
+ if (spark.master === 'yarn') {
+ return spark.master + '-' + spark.mode;
+ } else {
+ return spark.master;
+ }
+ })
+ .transform('spark', 'mode', function(spark) {
+ if (spark.master && spark.master.indexOf('yarn') != '-1') {
+ return spark.mode;
+ } else {
+ return null;
+ }
+ })
+ .transform('spark.name', 'name')
+ .transform('spark.class', 'class')
+ .transform('spark.jar', 'jar')
+ .transform('spark.sparkOptions', 'spark-opts')
+ .transform('spark.arg', 'arg');
+
+ var workflowNameCheck = function(workflow) {
+ if (workflow.engine === 'spark') {
+ return null;
+ } else {
+ return workflow.name;
+ }
+ }
+
var transform = transformerFactory
.transform('name', 'process._name')
.transform('tags', 'process.tags', keyValuePairs)
-
-
-
.transform('clusters', 'process.clusters.cluster', function(clusters) {
return clusters.map(function(cluster) {
return clusterTransform.apply(cluster, {});
@@ -234,10 +382,25 @@
return outputTransform.apply(output, {});
});
})
- .transform('workflow.name', 'process.workflow._name')
+ .transform('properties', 'process.properties.property', function(properties) {
+ properties = properties.filter(emptyProperty);
+ if(properties.length === 0) {
+ return null;
+ } else {
+ return properties.map(function(property) {
+ return propertyTransform.apply(property, {});
+ });
+ }
+ })
+ .transform('workflow', 'process.workflow._name', workflowNameCheck)
.transform('workflow.version', 'process.workflow._version')
.transform('workflow.engine', 'process.workflow._engine')
.transform('workflow.path', 'process.workflow._path')
+ .transform('workflow', 'process.spark-attributes', function(workflow) {
+ if (workflow.engine === 'spark') {
+ return sparkTransform.apply(workflow, {});
+ }
+ })
.transform('retry.policy', 'process.retry._policy')
.transform('retry.delay', 'process.retry._delay', frequencyToString)
.transform('retry.attempts', 'process.retry._attempts')
@@ -251,31 +414,149 @@
}
+ function preSerializeDatasource (datasource, transformerFactory) {
+ var credentialPasswordTextTransform = transformerFactory
+ .transform('type', '_type')
+ .transform('userName', 'userName')
+ .transform('passwordText', 'passwordText', function(value) {
+ return value;
+ });
+
+ var credentialPasswordFileTransform = transformerFactory
+ .transform('type', '_type')
+ .transform('userName', 'userName')
+ .transform('passwordFile', 'passwordFile');
+
+ var credentialPasswordAliasTransform = transformerFactory
+ .transform('type', '_type')
+ .transform('userName', 'userName')
+ .transform('passwordAlias', 'passwordAlias.alias')
+ .transform('providerPath', 'passwordAlias.providerPath');
+
+ var credentialTransform = function(credential) {
+ if (credential.type == 'password-text') {
+ return credentialPasswordTextTransform.apply(credential, {});
+ } if (credential.type == 'password-file') {
+ return credentialPasswordFileTransform.apply(credential, {});
+ } if (credential.type == 'password-alias') {
+ return credentialPasswordAliasTransform.apply(credential, {});
+ } else {
+ return null;
+ }
+ };
+
+ var interfaceTransform = transformerFactory
+ .transform('type', '_type')
+ .transform('endpoint', '_endpoint')
+ .transform('credential.type', 'credential._type')
+ .transform('credential', 'credential', credentialTransform);
+
+ var propertyTransform = transformerFactory
+ .transform('name', '_name')
+ .transform('value', '_value', function(value) {
+ return value;
+ });
+ var driverJarsTransform = function(driverJars){
+ var filtered = driverJars.filter(function(jar){
+ return jar.value && jar.value.trim().length > 0;
+ });
+ if(filtered.length > 0){
+ return filtered.map(function(jar){
+ return jar.value;
+ });
+ }
+ };
+ var transform = transformerFactory
+ .transform('colo', 'datasource._colo')
+ .transform('description', 'datasource._description')
+ .transform('type', 'datasource._type')
+ .transform('name', 'datasource._name')
+ .transform('tags', 'datasource.tags', keyValuePairs)
+ .transform('interfaces', 'datasource.interfaces.interface', function(datasourceInterfaces) {
+ return datasourceInterfaces.interfaces.length==0 ? null
+ : datasourceInterfaces.interfaces.map(function(datasourceInterface) {
+ return interfaceTransform.apply(datasourceInterface, {});
+ });
+ })
+ .transform('interfaces.credential', 'datasource.interfaces.credential', credentialTransform)
+ .transform('driver.clazz', 'datasource.driver.clazz')
+ .transform('driver.jar', 'datasource.driver.jar',driverJarsTransform)
+ .transform('allProperties', 'datasource.properties.property', function(properties) {
+ properties = properties.filter(emptyValue).filter(emptyFrequency);
+ return properties.length==0 ? null : properties.map(function(property) {
+ return propertyTransform.apply(property, {});
+ });
+ })
+ .transform('ACL', 'datasource.ACL', emptyElement)
+ .transform('ACL.owner', 'datasource.ACL._owner')
+ .transform('ACL.group', 'datasource.ACL._group')
+ .transform('ACL.permission', 'datasource.ACL._permission')
+
+ return transform.apply(datasource, new EntityModel('datasource'));
+
+ }
+
+ function preDeserializeCluster(clusterModel, transformerFactory) {
+
+ var cluster = EntityFactory.newClusterEntity();
+
+ var transform = transformerFactory
+ .transform('_name', 'name')
+ .transform('_colo','colo')
+ .transform('_description', 'description')
+ .transform('tags', 'tags', parseKeyValuePairs)
+ .transform('ACL._owner','ACL.owner')
+ .transform('ACL._group','ACL.group')
+ .transform('ACL._permission','ACL.permission')
+ .transform('locations.location', 'locations', parseClusterLocations)
+ .transform('properties.property', 'properties', parseClusterProperties)
+ .transform('interfaces.interface', 'interfaces', parseClusterInterfaces);
+
+ return transform.apply(angular.copy(clusterModel.cluster), cluster);
+ }
+
function preDeserializeFeed(feedModel, transformerFactory) {
var feed = EntityFactory.newFeed();
- feed.storage.fileSystem.active = false;
var clusterTransform = transformerFactory
.transform('_name', 'name')
.transform('_type', 'type')
+ .transform('_partition', 'partition')
.transform('validity._start', 'validity.start.date', (function () { feedTz = feedModel.feed.timezone; return importDateFeed; }()))
.transform('validity._start', 'validity.start.time', importDateFeed)
.transform('validity._end', 'validity.end.date', importDateFeed)
.transform('validity._end', 'validity.end.time', importDateFeed)
.transform('retention._limit', 'retention', parseFrequency)
.transform('retention._action', 'retention.action')
- .transform('locations', 'storage.fileSystem.active', parseBoolean)
.transform('locations.location', 'storage.fileSystem.locations', parseLocations)
- .transform('table', 'storage.catalog.active', parseBoolean)
.transform('table._uri', 'storage.catalog.catalogTable.uri')
;
+ var fieldTransform = transformerFactory
+ .transform('field', 'field');
+ var dataSourceTransformImport = transformerFactory
+ .transform('_name','name')
+ .transform('_tableName','tableName')
+ .transform('extract._type','extract.type')
+ .transform('extract.mergepolicy','extract.mergepolicy')
+ .transform('fields.excludes.field','excludesCSV', parseFields)
+ .transform('fields.includes.field','includesCSV', parseFields)
+ ;
+
+ var dataSourceTransformExport = transformerFactory
+ .transform('_name','name')
+ .transform('_tableName','tableName')
+ .transform('load._type','load.type')
+ .transform('fields.includes.field','includesCSV', parseFields)
+ .transform('fields.excludes.field','excludesCSV', parseFields)
+ ;
var transform = transformerFactory
.transform('_name', 'name')
.transform('_description', 'description')
.transform('tags', 'tags', parseKeyValuePairs)
.transform('groups','groups')
+ .transform('availabilityFlag', 'availabilityFlag')
.transform('ACL._owner','ACL.owner')
.transform('ACL._group','ACL.group')
.transform('ACL._permission','ACL.permission')
@@ -284,15 +565,44 @@
.transform('frequency','frequency', parseFrequency)
.transform('late-arrival','lateArrival.active', parseBoolean)
.transform('late-arrival._cut-off','lateArrival.cutOff', parseFrequency)
- .transform('availabilityFlag', 'availabilityFlag')
.transform('properties.property', 'customProperties', parseProperties(isCustomProperty, EntityFactory.newFeedCustomProperties()))
.transform('properties.property', 'properties', parseProperties(isFalconProperty, EntityFactory.newFeedProperties()))
- .transform('locations', 'storage.fileSystem.active', parseBoolean)
.transform('locations.location', 'storage.fileSystem.locations', parseLocations)
- .transform('table', 'storage.catalog.active', parseBoolean)
.transform('table._uri', 'storage.catalog.catalogTable.uri')
- .transform('clusters.cluster', 'clusters', parseClusters(clusterTransform))
- .transform('timezone', 'timezone');
+ .transform('partitions.partition', 'partitions', parsePartitions)
+ .transform('clusters.cluster', 'clusters', parseClusters(feed,clusterTransform))
+ .transform('clusters.cluster', 'datasources', parseFeedDatasources)
+ .transform('timezone', 'timezone')
+ ;
+
+ function parseFeedDatasources(clusters) {
+ if (clusters.length > 0) {
+ var cluster = clusters[0];
+ if (cluster.import) {
+ feed.dataTransferType = 'import';
+ feed.import = { 'source' : dataSourceTransformImport.apply(cluster.import.source, {}) };
+ if(cluster._type ==='source' && feed.storage.catalog.catalogTable.uri !== null){
+ feed.targetClusterLocationType = 'hive';
+ } else {
+ feed.targetClusterLocationType = 'hdfs';
+ }
+ } else if (clusters[0].export) {
+ feed.dataTransferType = 'export';
+ feed.export = { 'target' : dataSourceTransformExport.apply(clusters[0].export.target, {}) };
+ if(cluster._type ==='source' && feed.storage.catalog.catalogTable.uri !== null){
+ feed.targetClusterLocationType = 'hive';
+ } else {
+ feed.targetClusterLocationType = 'hdfs';
+ }
+ }
+ }
+ return null;
+ }
+
+ function parseFields(fields) {
+ return $.isArray(fields) ? fields.join() : fields;
+ }
+
return transform.apply(angular.copy(feedModel.feed), feed);
}
@@ -319,6 +629,19 @@
.transform('_feed', 'feed')
.transform('_instance', 'outputInstance');
+ var propertyTransform = transformerFactory
+ .transform('_name', 'name')
+ .transform('_value', 'value');
+
+ var sparkTransform = transformerFactory
+ .transform('master', 'master')
+ .transform('mode', 'mode')
+ .transform('name', 'name')
+ .transform('class', 'class')
+ .transform('jar', 'jar')
+ .transform('spark-opts', 'sparkOptions')
+ .transform('arg', 'arg');
+
var transform = transformerFactory
.transform('_name', 'name')
.transform('tags', 'tags', parseKeyValuePairs)
@@ -326,6 +649,7 @@
.transform('workflow._version', 'workflow.version')
.transform('workflow._engine', 'workflow.engine')
.transform('workflow._path', 'workflow.path')
+ .transform('spark-attributes', 'workflow.spark', parseSparkProperties(sparkTransform))
.transform('timezone', 'timezone')
.transform('frequency','frequency', parseFrequency)
.transform('parallel','parallel')
@@ -336,6 +660,7 @@
.transform('clusters.cluster', 'clusters', parseClusters(clusterTransform))
.transform('inputs.input', 'inputs', parseInputs(inputTransform))
.transform('outputs.output', 'outputs', parseOutputs(outputTransform))
+ .transform('properties.property', 'properties', parseProperties)
.transform('ACL._owner','ACL.owner')
.transform('ACL._group','ACL.group')
.transform('ACL._permission','ACL.permission');
@@ -348,9 +673,82 @@
};
}
+ function parseProperties(properties) {
+ return $.isArray(properties) ? properties.map(parseProperty) : [parseProperty(properties)];
+ }
+
+ function parseProperty(property) {
+ return EntityFactory.newProperty(property._name, property._value);
+ }
+
+ function parseSparkProperties(transform) {
+ return function(sparkAttributes) {
+ if (sparkAttributes.master && sparkAttributes.master.indexOf('yarn') !== '-1') {
+ sparkAttributes.master = 'yarn';
+ }
+ return sparkTransform.apply(sparkAttributes, EntityFactory.newSparkAttributes());
+ };
+ }
+
return transform.apply(angular.copy(processModel.process), process);
}
+ function preDeserializeDatasource(datasourceModel, transformerFactory) {
+ var datasource = EntityFactory.newDatasource();
+
+ function parseProperty(property) {
+ return EntityFactory.newProperty(property._name, property._value);
+ }
+
+ function parseProperties(filterCallback) {
+ return function(properties) {
+ var result = filter(properties, filterCallback).map(parseProperty);
+ return result;
+ };
+ }
+
+ function parseInterface(datasourceinterface) {
+ var interfaceTransform = transformerFactory
+ .transform('_type', 'type')
+ .transform('_endpoint', 'endpoint')
+ .transform('credential._type', 'credential.type')
+ .transform('credential.userName', 'credential.userName')
+ .transform('credential.passwordText', 'credential.passwordText')
+ .transform('credential.passwordFile', 'credential.passwordFile')
+ .transform('credential.passwordAlias.alias', 'credential.passwordAlias')
+ .transform('credential.passwordAlias.providerPath', 'credential.providerPath');
+
+ return interfaceTransform.apply(datasourceinterface, EntityFactory.newDatasourceInterface());
+ }
+
+ function parseInterfaces(interfaces) {
+ return $.isArray(interfaces) ? interfaces.map(parseInterface) : [parseInterface(interfaces)];
+ }
+ function parseDriverJar(jar){
+ return {value : jar};
+ }
+ function parseDriverJars(jars) {
+ return $.isArray(jars) ? jars.map(parseDriverJar) : [parseDriverJar(jars)];
+ }
+ var transform = transformerFactory
+ .transform('_name', 'name')
+ .transform('_description', 'description')
+ .transform('tags', 'tags', parseKeyValuePairs)
+ .transform('_colo','colo')
+ .transform('_type','type')
+ .transform('ACL._owner','ACL.owner')
+ .transform('ACL._group','ACL.group')
+ .transform('ACL._permission','ACL.permission')
+ .transform('driver.clazz','driver.clazz')
+ .transform('driver.jar','driver.jar',parseDriverJars)
+ .transform('interfaces.interface', 'interfaces.interfaces', parseInterfaces)
+ .transform('properties.property', 'properties', parseProperties(isDatasourceProperty))
+ .transform('properties.property', 'customProperties', parseProperties(isCustomDatasourceProperty));
+
+ return transform.apply(angular.copy(datasourceModel.datasource), datasource);
+
+ }
+
function parseDate(input) {
var dateComponent = (input.split('T')[0]).split('-');
return newUtcDate(dateComponent[0], dateComponent[1], dateComponent[2]);
@@ -361,9 +759,39 @@
return newUtcTime(timeComponent[0], timeComponent[1]);
}
- function parseClusters(transform) {
+ function parseClusters(feed, transform) {
return function(clusters) {
+ if (clusters.length > 0 && clusters[0] === "") {
+ return null;
+ }
var result = clusters.map(parseCluster(transform));
+ result.forEach(function(cluster){
+ if (cluster.type ==='target') {
+ feed.enableFeedReplication = true;
+ }
+ if(cluster.type ==='source' && cluster.storage.catalog){
+ feed.sourceClusterLocationType = 'hive';
+ }
+ if(cluster.type ==='target' && cluster.storage.catalog){
+ feed.targetClusterLocationType = 'hive';
+ }
+ if(cluster.type ==='source' && cluster.storage.fileSystem){
+ feed.sourceClusterLocationType = 'hdfs';
+ }
+ if(cluster.type ==='target' && cluster.storage.fileSystem){
+ feed.targetClusterLocationType = 'hdfs';
+ }
+ });
+ if (feed.sourceClusterLocationType === 'hive'
+ && (!feed.targetClusterLocationType || feed.targetClusterLocationType === 'hive')) {
+ feed.dataTransferType = 'hive';
+ } else if (feed.sourceClusterLocationType === 'hdfs'
+ && (!feed.targetClusterLocationType || feed.targetClusterLocationType === 'hdfs')) {
+ feed.dataTransferType = 'hdfs';
+ }
+ if (!feed.targetClusterLocationType && (feed.dataTransferType === 'hdfs' || feed.dataTransferType === 'hive')) {
+ feed.targetClusterLocationType = feed.dataTransferType;
+ }
selectFirstSourceCluster(result);
return result;
};
@@ -396,7 +824,7 @@
function parseCluster(transform) {
return function(input) {
var cluster = EntityFactory.newCluster('target', false);
- cluster.storage.fileSystem.active = false;
+ //cluster.storage.fileSystem.active = false;
return transform.apply(input, cluster);
};
}
@@ -477,6 +905,39 @@
return EntityFactory.newLocation(location._type, location._path);
}
+ function parseClusterLocations(locations) {
+ return $.isArray(locations) ? locations.map(parseClusterLocation) : [parseClusterLocation(locations)];
+ }
+
+ function parseClusterLocation(location) {
+ return EntityFactory.newClusterLocation(location._name, location._path);
+ }
+
+ function parseClusterProperties(properties) {
+ return $.isArray(properties) ? properties.map(parseClusterProperty) : [parseClusterProperty(properties)];
+ }
+
+ function parseClusterProperty(property) {
+ return EntityFactory.newEntry(property._name, property._value);
+ }
+
+ function parseClusterInterfaces(interfaces) {
+ return $.isArray(interfaces) ? interfaces.map(parseClusterInterface) : [parseClusterInterface(interfaces)];
+ }
+
+ function parseClusterInterface(clusterInterface) {
+ return EntityFactory.newClusterInterface(
+ clusterInterface._type, clusterInterface._endpoint, clusterInterface._version);
+ }
+
+ function parsePartitions(partitions) {
+ return $.isArray(partitions) ? partitions.map(parsePartititon) : [parsePartititon(partitions)];
+ }
+
+ function parsePartititon(partition) {
+ return EntityFactory.newPartition(partition._name);
+ }
+
function indexBy(array, property) {
var map = {};
@@ -507,7 +968,6 @@
mapBandwidthKB: true
};
-
function isCustomProperty(property) {
return !falconProperties[property._name];
}
@@ -516,5 +976,18 @@
return falconProperties[property._name];
}
+ var datasourceProperties = {
+ parameterFile: true,
+ verboseMode: true,
+ directMode: true
+ };
+
+ function isCustomDatasourceProperty(property) {
+ return !datasourceProperties[property._name];
+ }
+
+ function isDatasourceProperty(property) {
+ return datasourceProperties[property._name];
+ }
-})();
\ No newline at end of file
+})();
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/js/services/extension/extension-serializer.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/extension/extension-serializer.js b/falcon-ui/app/js/services/extension/extension-serializer.js
new file mode 100644
index 0000000..3f689b4
--- /dev/null
+++ b/falcon-ui/app/js/services/extension/extension-serializer.js
@@ -0,0 +1,415 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+(function () {
+ 'use strict';
+ var module = angular.module('app.services.extension.serializer',
+ ['app.services',
+ 'app.services.entity.factory',
+ 'app.services.entity.model']);
+
+ module.factory('ExtensionSerializer', ['EntityFactory', 'DateHelper', 'EntityModel',
+ function(EntityFactory, DateHelper, EntityModel) {
+
+ var convertTags = function (tagsArray) {
+ var result = [];
+ tagsArray.forEach(function(element) {
+ if(element.key && element.value) {
+ result.push(element.key + "=" + element.value);
+ }
+ });
+ result = result.join(",");
+ return result;
+ };
+
+ var convertObjectToString = function (obj) {
+ var str = '';
+ for (var key in obj) {
+ if (obj.hasOwnProperty(key)) {
+ str += key + '=' + obj[key] + '\n';
+ }
+ }
+ return str;
+ };
+
+ var serializeExtensionCommonProperties = function(extension) {
+ var extensionProps = {};
+ extensionProps.jobName = extension.name;
+ extensionProps.jobValidityStart = DateHelper.createISO(extension.validity.start.date,
+ extension.validity.start.time, extension.validity.timezone);
+ extensionProps.jobValidityEnd = DateHelper.createISO(extension.validity.end.date,
+ extension.validity.end.time, extension.validity.timezone);
+ extensionProps.jobFrequency = extension.frequency.unit + '(' + extension.frequency.quantity + ')';
+ extensionProps.jobTimezone = extension.validity.timezone;
+ extensionProps.jobTags = convertTags(extension.tags);
+ extensionProps.jobRetryPolicy = extension.retry.policy;
+ extensionProps.jobRetryDelay = extension.retry.delay.unit + '(' + extension.retry.delay.quantity + ')';
+ extensionProps.jobRetryAttempts = extension.retry.attempts;
+ extensionProps.jobAclOwner = extension.ACL.owner;
+ extensionProps.jobAclGroup = extension.ACL.group;
+ extensionProps.jobAclPermission = extension.ACL.permission;
+ extensionProps.tdeEncryptionEnabled = extensionProps.tdeEncryptionEnabled;
+
+ extensionProps.sourceCluster = extension.source.cluster;
+ extensionProps.targetCluster = extension.target.cluster;
+ if (extension.runOn === 'source') {
+ extensionProps.jobClusterName = extension.source.cluster;
+ } else if (extension.runOn === 'target') {
+ extensionProps.jobClusterName = extension.target.cluster;
+ }
+ if (extension.alerts.length > 0) {
+ extensionProps.jobNotificationType = 'email';
+ extensionProps.jobNotificationReceivers = extension.alerts.join();
+ }
+ return extensionProps;
+ };
+
+ var serializeSnapshotExtensionProperties = function(snapshot) {
+ var snapshotProps = serializeExtensionCommonProperties(snapshot);
+ snapshotProps.sourceSnapshotDir = snapshot.source.directoryPath.trim();
+ snapshotProps.targetSnapshotDir = snapshot.target.directoryPath.trim();
+ snapshotProps.sourceSnapshotRetentionAgeLimit = snapshot.source.deleteFrequency.unit
+ + '(' + snapshot.source.deleteFrequency.quantity + ')';
+ snapshotProps.targetSnapshotRetentionAgeLimit = snapshot.target.deleteFrequency.unit
+ + '(' + snapshot.target.deleteFrequency.quantity + ')';
+ snapshotProps.sourceSnapshotRetentionNumber = snapshot.source.retentionNumber;
+ snapshotProps.targetSnapshotRetentionNumber = snapshot.target.retentionNumber;
+ if (snapshot.allocation) {
+ if (snapshot.allocation.distcpMaxMaps) {
+ snapshotProps.distcpMaxMaps = snapshot.allocation.distcpMaxMaps;
+ }
+ if (snapshot.allocation.distcpMapBandwidth) {
+ snapshotProps.distcpMapBandwidth = snapshot.allocation.distcpMapBandwidth;
+ }
+ }
+
+ return snapshotProps;
+ };
+
+ var serializeHDFSExtensionProperties = function(hdfsExtension, secureMode) {
+ var hdfsExtensionProps = serializeExtensionCommonProperties(hdfsExtension);
+ hdfsExtensionProps.sourceDir = hdfsExtension.source.path.trim();
+ hdfsExtensionProps.targetDir = hdfsExtension.target.path.trim();
+ if (hdfsExtension.allocation.hdfs) {
+ if (hdfsExtension.allocation.hdfs.distcpMaxMaps) {
+ hdfsExtensionProps.distcpMaxMaps = hdfsExtension.allocation.hdfs.distcpMaxMaps;
+ }
+ if (hdfsExtension.allocation.hdfs.distcpMapBandwidth) {
+ hdfsExtensionProps.distcpMapBandwidth = hdfsExtension.allocation.hdfs.distcpMapBandwidth;
+ }
+ }
+ return hdfsExtensionProps;
+ };
+
+ var serializeHiveExtensionProperties = function(hiveEntension, secureMode) {
+ var hiveEntensionProps = serializeExtensionCommonProperties(hiveEntension);
+ if (hiveEntension.allocation.hive) {
+ if (hiveEntension.allocation.hive.distcpMaxMaps) {
+ hiveEntensionProps.distcpMaxMaps = hiveEntension.allocation.hive.distcpMaxMaps;
+ }
+ if (hiveEntension.allocation.hive.distcpMapBandwidth) {
+ hiveEntensionProps.distcpMapBandwidth = hiveEntension.allocation.hive.distcpMapBandwidth;
+ }
+ if (hiveEntension.allocation.hive.maxEvents) {
+ hiveEntensionProps.maxEvents = hiveEntension.allocation.hive.maxEvents;
+ }
+ if (hiveEntension.allocation.hive.replicationMaxMaps) {
+ hiveEntensionProps.replicationMaxMaps = hiveEntension.allocation.hive.replicationMaxMaps;
+ }
+ }
+
+ hiveEntensionProps.sourceDatabases = hiveEntension.source.hiveDatabases;
+ hiveEntensionProps.sourceTables = (hiveEntension.source.hiveDatabaseType === "databases")
+ ? "*" : hiveEntension.source.hiveTables;
+
+ hiveEntensionProps.sourceStagingPath = hiveEntension.hiveOptions.source.stagingPath
+ hiveEntensionProps.sourceHiveServer2Uri = hiveEntension.hiveOptions.source.hiveServerToEndpoint;
+ hiveEntensionProps.targetStagingPath = hiveEntension.hiveOptions.target.stagingPath
+ hiveEntensionProps.targetHiveServer2Uri = hiveEntension.hiveOptions.target.hiveServerToEndpoint;
+
+ // Secure mode
+ if (secureMode) {
+ hiveEntensionProps.sourceMetastoreUri = hiveEntension.source.hiveMetastoreUri;
+ hiveEntensionProps.sourceHiveMetastoreKerberosPrincipal = hiveEntension.source.hiveMetastoreKerberosPrincipal;
+ hiveEntensionProps.sourceHive2KerberosPrincipal = hiveEntension.source.hive2KerberosPrincipal;
+ hiveEntensionProps.targetMetastoreUri = hiveEntension.target.hiveMetastoreUri;
+ hiveEntensionProps.targetHiveMetastoreKerberosPrincipal = hiveEntension.target.hiveMetastoreKerberosPrincipal;
+ hiveEntensionProps.targetHive2KerberosPrincipal = hiveEntension.target.hive2KerberosPrincipal;
+ }
+
+ return hiveEntensionProps;
+ };
+
+ var serializeBasicExtensionModel = function(model, extensionObj) {
+
+ extensionObj.name = model.process._name;
+
+ extensionObj.retry.policy = model.process.retry._policy;
+ extensionObj.retry.attempts = model.process.retry._attempts;
+ extensionObj.retry.delay.quantity = (function () {
+ return parseInt(model.process.retry._delay.split('(')[1]);
+ }());
+ extensionObj.retry.delay.unit = (function () {
+ return model.process.retry._delay.split('(')[0];
+ }());
+
+ extensionObj.frequency.quantity = (function () {
+ return parseInt(model.process.frequency.split('(')[1]);
+ }());
+ extensionObj.frequency.unit = (function () {
+ return model.process.frequency.split('(')[0];
+ }());
+
+ // extensionObj.ACL.owner = model.process.ACL._owner;
+ // extensionObj.ACL.group = model.process.ACL._group;
+ // extensionObj.ACL.permissions = model.process.ACL._permission;
+
+ extensionObj.validity.timezone = model.process.timezone;
+ extensionObj.validity.start.date = DateHelper.importDate(
+ model.process.clusters.cluster[0].validity._start, model.process.timezone);
+ extensionObj.validity.start.time = DateHelper.importDate(
+ model.process.clusters.cluster[0].validity._start, model.process.timezone);
+ extensionObj.validity.end.date = DateHelper.importDate(
+ model.process.clusters.cluster[0].validity._end, model.process.timezone);
+ extensionObj.validity.end.time = DateHelper.importDate(
+ model.process.clusters.cluster[0].validity._end, model.process.timezone);
+
+ extensionObj.tags = (function () {
+ var array = [];
+ if(model.process && model.process.tags){
+ model.process.tags.split(',').forEach(function (fieldToSplit) {
+ var splittedString = fieldToSplit.split('=');
+ if (splittedString[0] != '_falcon_extension_name' && splittedString[0] != '_falcon_extension_job') {
+ array.push({key: splittedString[0], value: splittedString[1]});
+ }
+ });
+ }
+ return array;
+ }());
+
+ if(model.process && model.process.tags){
+ if (model.process.tags.indexOf('_falcon_extension_name=HDFS-MIRRORING') !== -1) {
+ extensionObj.type = 'HDFS';
+ } else if (model.process.tags.indexOf('_falcon_extension_name=HDFS-SNAPSHOT-MIRRORING') !== -1) {
+ extensionObj.type = 'snapshot';
+ } else if (model.process.tags.indexOf('_falcon_extension_name=HIVE-MIRRORING') !== -1) {
+ extensionObj.type = 'HIVE';
+ }
+ }
+
+ if (model.process.notification._to) {
+ extensionObj.alerts = (function () {
+ if (model.process.notification._to !== "NA") {
+ return model.process.notification._to.split(',');
+ } else {
+ return [];
+ }
+ }());
+ }
+
+ model.process.properties.property.forEach(function (item) {
+ if (item._name === 'targetCluster') {
+ extensionObj.target.cluster = item._value;
+ }
+ if (item._name === 'sourceCluster') {
+ extensionObj.source.cluster = item._value;
+ }
+ });
+
+ return extensionObj;
+ };
+
+ var serializeSnapshotExtensionModel = function(model, snapshotObj) {
+ snapshotObj = serializeBasicExtensionModel(model, snapshotObj);
+
+ model.process.properties.property.forEach(function (item) {
+ if (item._name === 'distcpMaxMaps') {
+ snapshotObj.allocation.distcpMaxMaps = item._value;
+ }
+ if (item._name === 'distcpMapBandwidth') {
+ snapshotObj.allocation.distcpMapBandwidth = item._value;
+ }
+ if (item._name === 'tdeEncryptionEnabled') {
+ if (item._value === 'true') {
+ snapshotObj.tdeEncryptionEnabled = true;
+ } else {
+ snapshotObj.tdeEncryptionEnabled = false;
+ }
+ }
+ if (item._name === 'sourceSnapshotDir') {
+ snapshotObj.source.directoryPath = item._value;
+ }
+ if (item._name === 'targetSnapshotDir') {
+ snapshotObj.target.directoryPath = item._value;
+ }
+ if (item._name === 'sourceSnapshotRetentionNumber') {
+ snapshotObj.source.retentionNumber = item._value;
+ }
+ if (item._name === 'targetSnapshotRetentionNumber') {
+ snapshotObj.target.retentionNumber = item._value;
+ }
+ if (item._name === 'sourceSnapshotRetentionAgeLimit') {
+ snapshotObj.source.deleteFrequency.quantity = (function () {
+ return parseInt(item._value.split('(')[1]);
+ }());
+ snapshotObj.source.deleteFrequency.unit = (function () {
+ return item._value.split('(')[0];
+ }());
+ }
+ if (item._name === 'targetSnapshotRetentionAgeLimit') {
+ snapshotObj.target.deleteFrequency.quantity = (function () {
+ return parseInt(item._value.split('(')[1]);
+ }());
+ snapshotObj.target.deleteFrequency.unit = (function () {
+ return item._value.split('(')[0];
+ }());
+ }
+ });
+
+ return snapshotObj;
+ };
+
+ var serializeHDFSExtensionModel = function(model, hdfsExtensionObj) {
+ hdfsExtensionObj = serializeBasicExtensionModel(model, hdfsExtensionObj);
+ model.process.properties.property.forEach(function (item) {
+ if (item._name === 'distcpMaxMaps') {
+ hdfsExtensionObj.allocation.hdfs.distcpMaxMaps = item._value;
+ }
+ if (item._name === 'distcpMapBandwidth') {
+ hdfsExtensionObj.allocation.hdfs.distcpMapBandwidth = item._value;
+ }
+ // if (item._name === 'tdeEncryptionEnabled') {
+ // if (item._value === 'true') {
+ // hdfsExtensionObj.tdeEncryptionEnabled = true;
+ // } else {
+ // hdfsExtensionObj.tdeEncryptionEnabled = false;
+ // }
+ // }
+ if (item._name === 'sourceDir') {
+ hdfsExtensionObj.source.path = item._value;
+ }
+ if (item._name === 'targetDir') {
+ hdfsExtensionObj.target.path = item._value;
+ }
+ });
+
+ return hdfsExtensionObj;
+ };
+
+ var serializeHiveExtensionModel = function(model, hiveExtensionObj, secureMode) {
+ hiveExtensionObj = serializeBasicExtensionModel(model, hiveExtensionObj);
+ model.process.properties.property.forEach(function (item) {
+ if (item._name === 'distcpMaxMaps') {
+ hiveExtensionObj.allocation.hive.distcpMaxMaps = item._value;
+ }
+ if (item._name === 'distcpMapBandwidth') {
+ hiveExtensionObj.allocation.hive.distcpMapBandwidth = item._value;
+ }
+ if (item._name === 'tdeEncryptionEnabled') {
+ if (item._value === 'true') {
+ hiveExtensionObj.tdeEncryptionEnabled = true;
+ } else {
+ hiveExtensionObj.tdeEncryptionEnabled = false;
+ }
+ }
+ if (item._name === 'replicationMaxMaps') {
+ hiveExtensionObj.allocation.hive.replicationMaxMaps = item._value;
+ }
+ if (item._name === 'maxEvents') {
+ hiveExtensionObj.allocation.hive.maxEvents = item._value;
+ }
+ if (item._name === 'sourceTables') {
+ if (item._value === "*") {
+ hiveExtensionObj.source.hiveDatabaseType = "databases";
+ } else {
+ hiveExtensionObj.source.hiveDatabaseType = "tables";
+ hiveExtensionObj.source.hiveTables = item._value;
+ }
+ }
+ if (item._name === 'sourceDatabases') {
+ hiveExtensionObj.source.hiveDatabases = item._value;
+ }
+
+ if (item._name === 'sourceStagingPath') {
+ hiveExtensionObj.hiveOptions.source.stagingPath = item._value;
+ }
+ if (item._name === 'targetStagingPath') {
+ hiveExtensionObj.hiveOptions.target.stagingPath = item._value;
+ }
+
+ if (item._name === 'sourceHiveServer2Uri') {
+ hiveExtensionObj.hiveOptions.source.hiveServerToEndpoint = item._value;
+ }
+ if (item._name === 'targetHiveServer2Uri') {
+ hiveExtensionObj.hiveOptions.target.hiveServerToEndpoint = item._value;
+ }
+
+ // Secure Mode Properties
+ if (secureMode) {
+ if(item._name === 'sourceMetastoreUri') {
+ hiveExtensionObj.source.hiveMetastoreUri = item._value;
+ }
+ if (item._name === 'sourceHiveMetastoreKerberosPrincipal') {
+ hiveExtensionObj.source.hiveMetastoreKerberosPrincipal = item._value;
+ }
+ if (item._name === 'sourceHive2KerberosPrincipal') {
+ hiveExtensionObj.source.hive2KerberosPrincipal = item._value;
+ }
+ if(item._name === 'targetMetastoreUri') {
+ hiveExtensionObj.target.hiveMetastoreUri = item._value;
+ }
+ if (item._name === 'targetHiveMetastoreKerberosPrincipal') {
+ hiveExtensionObj.target.hiveMetastoreKerberosPrincipal = item._value;
+ }
+ if (item._name === 'targetHive2KerberosPrincipal') {
+ hiveExtensionObj.target.hive2KerberosPrincipal = item._value;
+ }
+ }
+ });
+
+ return hiveExtensionObj;
+ };
+
+ var serializeExtensionProperties = function(extension, extensionType, secureMode) {
+ if (extensionType === 'snapshot') {
+ return serializeSnapshotExtensionProperties(extension);
+ } else if (extensionType === 'HDFS-MIRROR') {
+ return serializeHDFSExtensionProperties(extension, secureMode);
+ } else if (extensionType === 'HIVE-MIRROR') {
+ return serializeHiveExtensionProperties(extension, secureMode);
+ }
+ };
+
+ var serializeExtensionModel = function(extensionModel, extensionType, secureMode) {
+ if (extensionType === 'snapshot') {
+ return serializeSnapshotExtensionModel(extensionModel, EntityFactory.newEntity('snapshot'));
+ } else if (extensionType === 'hdfs-mirror') {
+ return serializeHDFSExtensionModel(extensionModel, EntityModel.datasetModel.UIModel);
+ } else if (extensionType === 'hive-mirror') {
+ return serializeHiveExtensionModel(extensionModel, EntityModel.datasetModel.UIModel, secureMode);
+ }
+ };
+
+ return {
+ serializeExtensionProperties: serializeExtensionProperties,
+ serializeExtensionModel: serializeExtensionModel,
+ convertObjectToString: convertObjectToString,
+ convertTags: convertTags
+ };
+
+ }]);
+})();
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/js/services/services.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/js/services/services.js b/falcon-ui/app/js/services/services.js
index 93d0482..41839fb 100644
--- a/falcon-ui/app/js/services/services.js
+++ b/falcon-ui/app/js/services/services.js
@@ -29,19 +29,19 @@
'app.services.entity.factory',
'app.services.entity.model',
'app.services.instance',
- 'app.services.server'
+ 'app.services.server',
+ 'app.services.entity.scheduler',
+ 'app.services.tooltip',
+ 'app.services.entity.details',
+ 'app.services.extension.serializer'
]);
services.factory('SpinnersFlag', function () {
return {
show: false,
- backShow: false
- };
- });
- services.factory('SpinnersFlag', function () {
- return {
- show: false,
- backShow: false
+ backShow: false,
+ saveShow: false,
+ validateShow: false
};
});
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/test/controllers/HeaderControllerSpec.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/test/controllers/HeaderControllerSpec.js b/falcon-ui/app/test/controllers/HeaderControllerSpec.js
index b3753db..9e2d986 100644
--- a/falcon-ui/app/test/controllers/HeaderControllerSpec.js
+++ b/falcon-ui/app/test/controllers/HeaderControllerSpec.js
@@ -29,30 +29,30 @@
interface:[
{
_type:"readonly",
- _endpoint:"hftp://sandbox.hortonworks.com:50070",
+ _endpoint:"hftp://localhost:50070",
_version:"2.2.0"
},
{
_type:"write",
- _endpoint:"hdfs://sandbox.hortonworks.com:8020",
+ _endpoint:"hdfs://localhost:8020",
_version:"2.2.0"
},
{
_type:"execute",
- _endpoint:"sandbox.hortonworks.com:8050",
+ _endpoint:"localhost:8050",
_version:"2.2.0"
},
{
_type:"workflow",
- _endpoint:"http://sandbox.hortonworks.com:11000/oozie/",
+ _endpoint:"http://localhost:11000/oozie/",
_version:"4.0.0"
},
{
_type:"messaging",
- _endpoint:"tcp://sandbox.hortonworks.com:61616?daemon=true",
+ _endpoint:"tcp://localhost:61616?daemon=true",
_version:"5.1.6"
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/test/controllers/cluster/cluster-moduleSpec.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/test/controllers/cluster/cluster-moduleSpec.js b/falcon-ui/app/test/controllers/cluster/cluster-moduleSpec.js
index 524f3eb..eef9630 100644
--- a/falcon-ui/app/test/controllers/cluster/cluster-moduleSpec.js
+++ b/falcon-ui/app/test/controllers/cluster/cluster-moduleSpec.js
@@ -26,12 +26,13 @@
entityModelArrangeMock = jasmine.createSpyObj('EntityModel', ['arrangeFieldsOrder']),
entityModel = {clusterModel :
{cluster:{tags: "",interfaces:{interface:[
- {_type:"readonly",_endpoint:"hftp://sandbox.hortonworks.com:50070",_version:"2.2.0"},
- {_type:"write",_endpoint:"hdfs://sandbox.hortonworks.com:8020",_version:"2.2.0"},
- {_type:"execute",_endpoint:"sandbox.hortonworks.com:8050",_version:"2.2.0"},
- {_type:"workflow",_endpoint:"http://sandbox.hortonworks.com:11000/oozie/",_version:"4.0.0"},
- {_type:"messaging",_endpoint:"tcp://sandbox.hortonworks.com:61616?daemon=true",_version:"5.1.6"}
- ]},locations:{location:[{_name: "staging", _path: ""},{_name: "temp", _path: ""},{_name: "working", _path: ""}]},
+ {_type:"readonly",_endpoint:"hftp://localhost:50070",_version:"2.2.0"},
+ {_type:"write",_endpoint:"hdfs://localhost:8020",_version:"2.2.0"},
+ {_type:"execute",_endpoint:"localhost:8050",_version:"2.2.0"},
+ {_type:"workflow",_endpoint:"http://localhost:11000/oozie/",_version:"4.0.0"},
+ {_type:"messaging",_endpoint:"tcp://localhost:61616?daemon=true",_version:"5.1.6"},
+ {_type:"spark",_endpoint:"",_version:""}
+ ]},locations:{location:[{_name: "staging", _path: ""},{_name: "temp", _path: "/tmp"},{_name: "working", _path: ""}]},
ACL: {_owner: "",_group: "",_permission: ""},properties: {property: [{ _name: "", _value: ""}]},
_xmlns:"uri:falcon:cluster:0.1",_name:"",_description:"",_colo:""},
}},
@@ -57,9 +58,16 @@
$scope: scope,
Falcon: falconServiceMock,
EntityModel: entityModel,
- $state: stateMock,
+ $state: {
+ current: {
+ name: 'main.forms.custer.general'
+ },
+ go: angular.noop
+ },
X2jsService: x2jsServiceMock,
- validationService:ValidationService
+ validationService:ValidationService,
+ $stateParams: {},
+ ClusterModel: {cluster : {}},
});
//
}));
@@ -73,8 +81,8 @@
expect(scope.secondStep).toEqual(false);
expect(scope.clusterEntity.clusterModel.cluster.properties.property).toEqual([{ _name: "", _value: ""}]);
- expect(scope.registry).toEqual({ check: false });
- expect(scope.registry).toEqual({ check: false });
+ // expect(scope.registry).toEqual({ check: true });
+ // expect(scope.registry).toEqual({ check: true });
});
});
describe('tags', function() {
@@ -137,7 +145,7 @@
it('should init with default locations and correct values', function() {
expect(scope.clusterEntity.clusterModel.cluster.locations.location).toEqual(
- [{ _name : 'staging', _path : '' }, { _name : 'temp', _path : '' }, { _name : 'working', _path : '' }, { _name : '', _path : '' }]
+ [{ _name : 'staging', _path : '' }, { _name : 'temp', _path : '/tmp' }, { _name : 'working', _path : '' }]
);
});
});
@@ -264,16 +272,16 @@
it('should delete registry interface only if not checked', function() {
scope.validations = validationService;
scope.clusterEntity.clusterModel.cluster.tags = "";
- expect(scope.registry.check).toBe(true);
- expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface.length).toEqual(6);
- expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface[5]).toEqual({ _type : 'registry', _endpoint : '', _version : '' });
+ expect(scope.registry.check).toBe(false);
+ expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface.length).toEqual(7);
+ //expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface[5]).toEqual({ _type : 'registry', _endpoint : 'thrift://<hostname>:9083', _version : '' });
scope.goSummaryStep();
- expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface[5]).toEqual({ _type : 'registry', _endpoint : '', _version : '' });
+ //expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface[5]).toEqual({ _type : 'registry', _endpoint : 'thrift://<hostname>:9083', _version : '' });
scope.registry.check = false;
scope.clusterEntity.clusterModel.cluster.ACL = { _owner : '', _group : '', _permission : '' };
scope.clusterEntity.clusterModel.cluster.tags = "";
scope.goSummaryStep();
- expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface[5]).toBeUndefined();
+ expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface[7]).toBeUndefined();
expect(scope.clusterEntity.clusterModel.cluster.interfaces.interface.length).toEqual(5);
});
@@ -333,4 +341,4 @@
});
});
-})();
\ No newline at end of file
+})();
http://git-wip-us.apache.org/repos/asf/falcon/blob/76dc2e18/falcon-ui/app/test/controllers/feed/FeedGeneralInformationControllerSpec.js
----------------------------------------------------------------------
diff --git a/falcon-ui/app/test/controllers/feed/FeedGeneralInformationControllerSpec.js b/falcon-ui/app/test/controllers/feed/FeedGeneralInformationControllerSpec.js
index 34c39cb..e89f088 100644
--- a/falcon-ui/app/test/controllers/feed/FeedGeneralInformationControllerSpec.js
+++ b/falcon-ui/app/test/controllers/feed/FeedGeneralInformationControllerSpec.js
@@ -32,6 +32,7 @@
controller = $controller('FeedGeneralInformationController', {
$scope: scope,
+ clustersList: [],
$state: {},
$filter: $filter
});