You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/06/28 05:57:33 UTC
[20/25] incubator-atlas git commit: ATLAS-1898: initial commit of ODF
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataSet.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataSet.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataSet.java
new file mode 100755
index 0000000..0b75eae
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataSet.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import io.swagger.annotations.ApiModel;
+
+// JSON
+@ApiModel(description="Metadata object representing a generic data set.")
+public abstract class DataSet extends MetaDataObject {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataStore.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataStore.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataStore.java
new file mode 100755
index 0000000..f4a11bd
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/DataStore.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import java.util.List;
+
+import org.apache.atlas.odf.api.metadata.MetaDataObjectReference;
+
+/**
+ * This class represents a metadataobject that references other metadataobjects
+ *
+ */
+public abstract class DataStore extends MetaDataObject {
+ private List<MetaDataObjectReference> connections;
+
+ public List<MetaDataObjectReference> getConnections() {
+ return connections;
+ }
+
+ public void setConnections(List<MetaDataObjectReference> connections) {
+ this.connections = connections;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Database.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Database.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Database.java
new file mode 100755
index 0000000..bbf7a0a
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Database.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+/**
+ * This class is a metadata object for a relational database
+ *
+ */
+public class Database extends DataStore {
+ private String dbType;
+
+ public String getDbType() {
+ return dbType;
+ }
+
+ public void setDbType(String dbType) {
+ this.dbType = dbType;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Document.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Document.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Document.java
new file mode 100755
index 0000000..ed6c0ef
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Document.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+/**
+ *
+ * This class represents a text document stored at a URL
+ *
+ */
+public class Document extends DataSet {
+ private String encoding = "UTF-8";
+ private String urlString;
+
+ public String getUrlString() {
+ return urlString;
+ }
+
+ public void setUrlString(String url) {
+ this.urlString = url;
+ }
+
+ public String getEncoding() {
+ return encoding;
+ }
+
+ public void setEncoding(String encoding) {
+ this.encoding = encoding;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnection.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnection.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnection.java
new file mode 100755
index 0000000..0a96fb1
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnection.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+/**
+ * This class represents a JDCB connection that can be used to access the contents of a dataset
+ *
+ */
+public class JDBCConnection extends Connection {
+ private String jdbcConnectionString;
+ private String user;
+ private String password;
+
+ public String getJdbcConnectionString() {
+ return jdbcConnectionString;
+ }
+
+ public void setJdbcConnectionString(String jdbcConnectionString) {
+ this.jdbcConnectionString = jdbcConnectionString;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnectionInfo.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnectionInfo.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnectionInfo.java
new file mode 100755
index 0000000..131f3f9
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/JDBCConnectionInfo.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+/**
+ *
+ * Provides all information for connecting to a JDBC data source
+ *
+ */
+@ApiModel(description="Object containing the information required to access a specific JDBC table.")
+public class JDBCConnectionInfo extends ConnectionInfo {
+
+ @ApiModelProperty(value="Table name", readOnly=true, required=true)
+ private String tableName;
+
+ @ApiModelProperty(value="Schema name", readOnly=true, required=true)
+ private String schemaName;
+
+ public String getTableName() {
+ return this.tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getSchemaName() {
+ return this.schemaName;
+ }
+
+ public void setSchemaName(String schemaName) {
+ this.schemaName = schemaName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataCache.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataCache.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataCache.java
new file mode 100755
index 0000000..ff2f47f
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataCache.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.atlas.odf.api.metadata.StoredMetaDataObject;
+
+import io.swagger.annotations.ApiModelProperty;
+
+public class MetaDataCache {
+
+ @ApiModelProperty(value="Cached metadata objects to be used by discovery services if access to the metadata store is not available", required=false)
+ private List<StoredMetaDataObject> metaDataObjects = new ArrayList<StoredMetaDataObject>();
+
+ @ApiModelProperty(value="Cached connection info objects to be used by discovery services if access to the metadata store is not available", required=false)
+ private List<ConnectionInfo> connectionInfoObjects = new ArrayList<ConnectionInfo>();
+
+ public List<StoredMetaDataObject> getMetaDataObjects() {
+ return metaDataObjects;
+ }
+
+ public void setMetaDataObjects(List<StoredMetaDataObject> metaDataObjects) {
+ this.metaDataObjects = metaDataObjects;
+ }
+
+ public List<ConnectionInfo> getConnectionInfoObjects() {
+ return this.connectionInfoObjects;
+ }
+
+ public void setConnectionInfoObjects(List<ConnectionInfo> connectionInfoObjects) {
+ this.connectionInfoObjects = connectionInfoObjects;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataObject.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataObject.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataObject.java
new file mode 100755
index 0000000..6152d51
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/MetaDataObject.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import java.util.List;
+
+import org.apache.atlas.odf.api.metadata.MetaDataObjectReference;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+// JSON
+/**
+ *
+ * A MetaDataObject is an object in a Metadata store, containing a reference describing its location and annotations that were created on this object
+ *
+ */
+@ApiModel(description="Metadata object representing a generic data set.")
+public abstract class MetaDataObject {
+
+ @ApiModelProperty(value="Reference to the object (generated)", readOnly=true, required=true)
+ private MetaDataObjectReference reference;
+
+ @ApiModelProperty(value="Description of the object", required=false)
+ private String description;
+
+ @ApiModelProperty(value="Name of the object", required=true)
+ private String name;
+
+ @ApiModelProperty(value="Java class represeting the object", hidden=true)
+ private String javaClass = this.getClass().getName(); // don't use JsonTypeInfo
+
+ private String originRef;
+
+ private List<String> replicaRefs;
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public MetaDataObjectReference getReference() {
+ return reference;
+ }
+
+ public void setReference(MetaDataObjectReference reference) {
+ this.reference = reference;
+ }
+
+ public String getJavaClass() {
+ return javaClass;
+ }
+
+ public void setJavaClass(String javaClass) {
+ this.javaClass = javaClass;
+ }
+
+ public String getOriginRef() {
+ return originRef;
+ }
+
+ public void setOriginRef(String originRef) {
+ this.originRef = originRef;
+ }
+
+ public List<String> getReplicaRefs() {
+ return replicaRefs;
+ }
+
+ public void setReplicaRefs(List<String> replicaRefs) {
+ this.replicaRefs = replicaRefs;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ProfilingAnnotation.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ProfilingAnnotation.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ProfilingAnnotation.java
new file mode 100755
index 0000000..8e6fcca
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/ProfilingAnnotation.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import org.apache.atlas.odf.api.metadata.MetaDataObjectReference;
+
+public class ProfilingAnnotation extends Annotation {
+
+ private MetaDataObjectReference profiledObject;
+
+ public MetaDataObjectReference getProfiledObject() {
+ return profiledObject;
+ }
+ public void setProfiledObject(MetaDataObjectReference annotatedObject) {
+ this.profiledObject = annotatedObject;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationalDataSet.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationalDataSet.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationalDataSet.java
new file mode 100755
index 0000000..e8656c5
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationalDataSet.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+
+/**
+ *
+ * This class represents a metadataobject that contains columns, eg. a database table or a CSV file
+ *
+ */
+public abstract class RelationalDataSet extends DataSet {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationshipAnnotation.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationshipAnnotation.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationshipAnnotation.java
new file mode 100755
index 0000000..924dadf
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/RelationshipAnnotation.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import java.util.List;
+
+import org.apache.atlas.odf.api.metadata.MetaDataObjectReference;
+
+public class RelationshipAnnotation extends Annotation {
+
+ private List<MetaDataObjectReference> relatedObjects;
+
+ public List<MetaDataObjectReference> getRelatedObjects() {
+ return relatedObjects;
+ }
+
+ public void setRelatedObjects(List<MetaDataObjectReference> relatedObjects) {
+ this.relatedObjects = relatedObjects;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Schema.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Schema.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Schema.java
new file mode 100755
index 0000000..e74bda4
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Schema.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+public class Schema extends MetaDataObject {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Table.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Table.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Table.java
new file mode 100755
index 0000000..633706d
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/Table.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+/**
+ *
+ * This class represents a database table metadataobject
+ *
+ */
+public class Table extends RelationalDataSet {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnection.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnection.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnection.java
new file mode 100755
index 0000000..3a1a968
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnection.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UnknownConnection extends Connection {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnectionInfo.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnectionInfo.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnectionInfo.java
new file mode 100755
index 0000000..57da8d3
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownConnectionInfo.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UnknownConnectionInfo extends ConnectionInfo {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataSet.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataSet.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataSet.java
new file mode 100755
index 0000000..948bf3f
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataSet.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UnknownDataSet extends DataSet {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataStore.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataStore.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataStore.java
new file mode 100755
index 0000000..d07537e
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownDataStore.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UnknownDataStore extends DataStore {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownRelationalDataSet.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownRelationalDataSet.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownRelationalDataSet.java
new file mode 100755
index 0000000..6993751
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/metadata/models/UnknownRelationalDataSet.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.metadata.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UnknownRelationalDataSet extends RelationalDataSet {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaConsumerConfig.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaConsumerConfig.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaConsumerConfig.java
new file mode 100755
index 0000000..08ca741
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaConsumerConfig.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings;
+
+import org.apache.atlas.odf.api.settings.validation.EnumValidator;
+import org.apache.atlas.odf.api.settings.validation.NumberPositiveValidator;
+import org.apache.atlas.odf.api.settings.validation.ValidationException;
+
+/*
+ * This class is final, because reflection is used to access getters / setters in order to merge. This doesn't work with inherited methods
+ */
+public final class KafkaConsumerConfig {
+ /*
+ * ############ !!!!!!!!!!!!!!!!!!! ###################
+ *
+ * Because of a jackson defect, JsonProperty annotations must be on all properties AND their getters and setters!
+ *
+ * https://github.com/FasterXML/jackson-module-scala/issues/197
+ */
+
+ private String offsetsStorage;
+
+ private Long zookeeperSessionTimeoutMs;
+
+ private Long zookeeperConnectionTimeoutMs;
+
+ public String getOffsetsStorage() {
+ return offsetsStorage;
+ }
+
+ public void setOffsetsStorage(String offsetsStorage) {
+ this.offsetsStorage = offsetsStorage;
+ }
+
+ public Long getZookeeperSessionTimeoutMs() {
+ return zookeeperSessionTimeoutMs;
+ }
+
+ public void setZookeeperSessionTimeoutMs(Long zookeeperSessionTimeoutMs) {
+ this.zookeeperSessionTimeoutMs = zookeeperSessionTimeoutMs;
+ }
+
+ public Long getZookeeperConnectionTimeoutMs() {
+ return zookeeperConnectionTimeoutMs;
+ }
+
+ public void setZookeeperConnectionTimeoutMs(Long zookeeperConnectionTimeoutMs) {
+ this.zookeeperConnectionTimeoutMs = zookeeperConnectionTimeoutMs;
+ }
+
+ public void validate() throws ValidationException {
+ if (getOffsetsStorage() != null) {
+ new EnumValidator("kafka", "zookeeper").validate("KafkaConsumerConfig.offsetsStorage", this.offsetsStorage);
+ }
+ if (getZookeeperConnectionTimeoutMs() != null) {
+ new NumberPositiveValidator().validate("KafkaConsumerConfig.zookeeperConnectionTimeoutMs", this.zookeeperConnectionTimeoutMs);
+ }
+ if (getZookeeperSessionTimeoutMs() != null) {
+ new NumberPositiveValidator().validate("KafkaConsumerConfig.zookeeperSessionTimeoutMs", this.zookeeperSessionTimeoutMs);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaMessagingConfiguration.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaMessagingConfiguration.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaMessagingConfiguration.java
new file mode 100755
index 0000000..3c1725e
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/KafkaMessagingConfiguration.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings;
+
+import org.apache.atlas.odf.api.settings.validation.NumberPositiveValidator;
+import org.apache.atlas.odf.api.settings.validation.ValidationException;
+
+public class KafkaMessagingConfiguration extends MessagingConfiguration {
+ private KafkaConsumerConfig kafkaConsumerConfig;
+
+ private Integer queueConsumerWaitMs;
+
+ private Integer kafkaBrokerTopicReplication;
+
+ public Integer getKafkaBrokerTopicReplication() {
+ return kafkaBrokerTopicReplication;
+ }
+
+ public void setKafkaBrokerTopicReplication(Integer kafkaBrokerTopicReplication) {
+ this.kafkaBrokerTopicReplication = kafkaBrokerTopicReplication;
+ }
+
+ public Integer getQueueConsumerWaitMs() {
+ return queueConsumerWaitMs;
+ }
+
+ public void setQueueConsumerWaitMs(Integer queueConsumerWaitMs) {
+ this.queueConsumerWaitMs = queueConsumerWaitMs;
+ }
+
+ public KafkaConsumerConfig getKafkaConsumerConfig() {
+ return kafkaConsumerConfig;
+ }
+
+ public void setKafkaConsumerConfig(KafkaConsumerConfig kafkaConsumerConfig) {
+ this.kafkaConsumerConfig = kafkaConsumerConfig;
+ }
+
+ public void validate() throws ValidationException {
+ super.validate();
+ if (this.getQueueConsumerWaitMs() != null) {
+ new NumberPositiveValidator().validate("ODFConfig.queueConsumerWaitMs", this.queueConsumerWaitMs);
+ }
+ if (this.getKafkaBrokerTopicReplication() != null) {
+ new NumberPositiveValidator().validate("ODFConfig.kafkaBrokerTopicReplication", this.kafkaBrokerTopicReplication);
+ }
+ if (this.getKafkaConsumerConfig() != null) {
+ this.kafkaConsumerConfig.validate();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/MessagingConfiguration.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/MessagingConfiguration.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/MessagingConfiguration.java
new file mode 100755
index 0000000..ba006e3
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/MessagingConfiguration.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.apache.atlas.odf.api.settings.validation.NumberPositiveValidator;
+import org.apache.atlas.odf.api.settings.validation.ValidationException;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+@ApiModel(description="Messaging configuration to be used for queuing requests.")
+@JsonTypeInfo(use = JsonTypeInfo.Id.CLASS, include = JsonTypeInfo.As.PROPERTY, property = "type")
+public abstract class MessagingConfiguration {
+ @ApiModelProperty(value="Time analysis requests are retained on the queue in milliseconds")
+ private Long analysisRequestRetentionMs;
+
+ public Long getAnalysisRequestRetentionMs() {
+ return analysisRequestRetentionMs;
+ }
+
+ public void setAnalysisRequestRetentionMs(Long analysisRequestRetentionMs) {
+ this.analysisRequestRetentionMs = analysisRequestRetentionMs;
+ }
+
+ public void validate() throws ValidationException {
+ if (this.getAnalysisRequestRetentionMs() != null) {
+ new NumberPositiveValidator().validate("ODFConfig.analysisRequestRetentionMs", this.analysisRequestRetentionMs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/ODFSettings.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/ODFSettings.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/ODFSettings.java
new file mode 100755
index 0000000..2124c54
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/ODFSettings.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings;
+
+import java.util.Map;
+import org.apache.atlas.odf.api.settings.validation.NumberPositiveValidator;
+import org.apache.atlas.odf.api.settings.validation.StringNotEmptyValidator;
+import org.apache.atlas.odf.api.settings.validation.ValidationException;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+/*
+ *
+ * This class is final, because reflection is used to access getters / setters in order to merge. This doesn't work with inherited methods
+ * Boolean properties must be of type Boolean instead of boolean in order to support null values which are required for merging later!
+ *
+ */
+@ApiModel(description="General ODF settings.")
+public final class ODFSettings {
+
+ /*
+ * ############ !!!!!!!!!!!!!!!!!!! ####################
+ *
+ * Because of a jackson defect, JsonProperty annotations must be on all properties AND their getters and setters!
+ *
+ * https://github.com/FasterXML/jackson-module-scala/issues/197
+ */
+
+ @ApiModelProperty(value="Polling interval for checking whether a discovery service is still running")
+ private Integer discoveryServiceWatcherWaitMs;
+
+ @ApiModelProperty(value="Unique id of the ODF instance")
+ private String instanceId;
+
+ @ApiModelProperty(value="ODF URL passed to discovery services for remote access to the metadata API")
+ private String odfUrl;
+
+ @ApiModelProperty(value="ODF user id passed to discovery services for remote access to the metadata API")
+ private String odfUser;
+
+ @ApiModelProperty(value="ODF password passed to discovery services for remote access to the metadata API")
+ private String odfPassword;
+
+ @ApiModelProperty(value = "ATLAS setting indicating if events regarding newly imported data sets should be consumed from me")
+ private Boolean consumeMessageHubEvents;
+
+ @ApiModelProperty(value = "ATLAS Messagehub VCAP_SERVICES value from Bluemix, e.g { \"messagehub\": [{\"name\" : \"...\",\n\"credentials\": {...}]")
+ private String atlasMessagehubVcap;
+
+ @ApiModelProperty(value="Indicates whether to reuse equivalent analysis requests that may be already queued rather that running the same analysis again")
+ private Boolean reuseRequests;
+
+ @ApiModelProperty(value="Messaging configuration to be used for queuing requests")
+ private MessagingConfiguration messagingConfiguration;
+
+ @ApiModelProperty(value="If set to true, ALL registered discovery services will be automatically issued when a new data set is imported")
+ private Boolean runAnalysisOnImport;
+
+ @ApiModelProperty(value="If set to true, ALL data sets will be automatically analyzed whenever a new discovery service is registered")
+ private Boolean runNewServicesOnRegistration;
+
+ @ApiModelProperty(value="User-defined configuration options for discovery services", required=true)
+ private Map<String, Object> userDefined;
+
+ @ApiModelProperty(value="Spark clusters to be used for running discovery services", required=true)
+ private SparkConfig sparkConfig;
+
+ @ApiModelProperty(value = "Set to true to propagate the created annotations for each analysis request to the metadata store")
+ private Boolean enableAnnotationPropagation;
+
+ public Boolean getEnableAnnotationPropagation() {
+ return enableAnnotationPropagation;
+ }
+
+ public void setEnableAnnotationPropagation(Boolean enableAnnotationPropagation) {
+ this.enableAnnotationPropagation = enableAnnotationPropagation;
+ }
+
+ public Boolean isReuseRequests() {
+ return reuseRequests;
+ }
+
+ public void setReuseRequests(Boolean reuseRequests) {
+ this.reuseRequests = reuseRequests;
+ }
+
+ public String getInstanceId() {
+ return this.instanceId;
+ }
+
+ public void setInstanceId(String instanceId) {
+ this.instanceId = instanceId;
+ }
+
+ public String getOdfUrl() {
+ return this.odfUrl;
+ }
+
+ public void setOdfUrl(String odfUrl) {
+ this.odfUrl = odfUrl;
+ }
+
+ public String getOdfUser() {
+ return this.odfUser;
+ }
+
+ public void setOdfUser(String odfUser) {
+ this.odfUser = odfUser;
+ }
+
+ public String getOdfPassword() {
+ return this.odfPassword;
+ }
+
+ public void setOdfPassword(String odfPassword) {
+ this.odfPassword = odfPassword;
+ }
+
+ public Integer getDiscoveryServiceWatcherWaitMs() {
+ return discoveryServiceWatcherWaitMs;
+ }
+
+ public void setDiscoveryServiceWatcherWaitMs(Integer discoveryServiceWatcherWaitMs) {
+ this.discoveryServiceWatcherWaitMs = discoveryServiceWatcherWaitMs;
+ }
+
+ public Boolean getRunAnalysisOnImport() {
+ return runAnalysisOnImport;
+ }
+
+ public void setRunAnalysisOnImport(Boolean runAnalysisOnImport) {
+ this.runAnalysisOnImport = runAnalysisOnImport;
+ }
+
+ public Boolean getRunNewServicesOnRegistration() {
+ return runNewServicesOnRegistration;
+ }
+
+ public void setRunNewServicesOnRegistration(Boolean runNewServicesOnRegistration) {
+ this.runNewServicesOnRegistration = runNewServicesOnRegistration;
+ }
+
+ public MessagingConfiguration getMessagingConfiguration() {
+ return messagingConfiguration;
+ }
+
+ public void setMessagingConfiguration(MessagingConfiguration messagingConfiguration) {
+ this.messagingConfiguration = messagingConfiguration;
+ }
+
+ public String getAtlasMessagehubVcap() {
+ return atlasMessagehubVcap;
+ }
+
+ public void setAtlasMessagehubVcap(String atlasMessagehubVcap) {
+ this.atlasMessagehubVcap = atlasMessagehubVcap;
+ }
+
+ public Map<String, Object> getUserDefined() {
+ return userDefined;
+ }
+
+ public Boolean getConsumeMessageHubEvents() {
+ return consumeMessageHubEvents;
+ }
+
+ public void setConsumeMessageHubEvents(Boolean consumeMessageHubEvents) {
+ this.consumeMessageHubEvents = consumeMessageHubEvents;
+ }
+
+ public void setUserDefined(Map<String, Object> userDefined) {
+ this.userDefined = userDefined;
+ }
+
+ public SparkConfig getSparkConfig() {
+ return sparkConfig;
+ }
+
+ public void setSparkConfig(SparkConfig sparkConfig) {
+ this.sparkConfig = sparkConfig;
+ }
+
+ public void validate() throws ValidationException {
+ new StringNotEmptyValidator().validate("ODFConfig.instanceId", instanceId);
+
+ if (this.getDiscoveryServiceWatcherWaitMs() != null) {
+ new NumberPositiveValidator().validate("ODFConfig.discoveryServiceWatcherWaitMs", this.discoveryServiceWatcherWaitMs);
+ }
+
+ if (this.messagingConfiguration != null) {
+ this.messagingConfiguration.validate();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SettingsManager.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SettingsManager.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SettingsManager.java
new file mode 100755
index 0000000..9c300b9
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SettingsManager.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings;
+
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.atlas.odf.api.settings.validation.ValidationException;
+
+/**
+*
+* External Java API for reading and updating ODF settings
+*
+*/
+public interface SettingsManager {
+
+ /**
+ * Retrieve Kafka consumer properties
+ * @return Current Kafka consumer properties
+ */
+ public Properties getKafkaConsumerProperties();
+
+ /**
+ * Retrieve Kafka producer properties
+ * @return Current Kafka producer properties
+ */
+ public Properties getKafkaProducerProperties();
+
+ /**
+ * Retrieve overall ODF settings including plain passwords
+ * @return Current ODF settings
+ */
+ public ODFSettings getODFSettings();
+
+ /**
+ * Retrieve overall ODF settings with hidden passwords
+ * @return Current ODF settings
+ */
+ public ODFSettings getODFSettingsHidePasswords();
+
+ /**
+ * Update ODF settings
+ *
+ * Passwords provided as plain text will be encrypted. If HIDDEN_PASSWORD_IDENTIFIER
+ * is provided instead of a password, the stored password will remain unchanged.
+ *
+ * @param Updated ODF settings
+ */
+ public void updateODFSettings(ODFSettings update) throws ValidationException;
+
+ /**
+ * Reset ODF settings to the defaults
+ */
+ public void resetODFSettings();
+
+ /**
+ * Retrieve user defined ODF properties
+ * @return Map of user defined ODF properties
+ */
+ public Map<String, Object> getUserDefinedConfig();
+
+ /**
+ * Update user defined ODF properties
+ * @param Map of user defined ODF properties
+ * @throws ValidationException
+ */
+ public void updateUserDefined(Map<String, Object> update) throws ValidationException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SparkConfig.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SparkConfig.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SparkConfig.java
new file mode 100755
index 0000000..5c3694c
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/SparkConfig.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+//JSON
+@ApiModel(description = "Configuration of Spark cluster.")
+public class SparkConfig {
+
+ @ApiModelProperty(value = "Master URL of the Spark cluster", required = true)
+ private String clusterMasterUrl = null;
+
+ @ApiModelProperty(value="Custom Spark configuration options", required=false)
+ private Map<String, Object> configs = new HashMap<>();
+
+ public String getClusterMasterUrl() {
+ return this.clusterMasterUrl;
+ }
+
+ public void setClusterMasterUrl(String clusterMasterUrl) {
+ this.clusterMasterUrl = clusterMasterUrl;
+ }
+
+ @JsonAnyGetter
+ public Map<String, Object> getConfigs() {
+ return this.configs;
+ }
+
+ @JsonAnySetter
+ public void setConfig(String name, Object value) {
+ this.configs.put(name, value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/EnumValidator.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/EnumValidator.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/EnumValidator.java
new file mode 100755
index 0000000..c6c365f
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/EnumValidator.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings.validation;
+
+import java.text.MessageFormat;
+
+public class EnumValidator implements PropertyValidator {
+
+ String[] validValues = new String[0];
+
+ public EnumValidator(String... validValues) {
+ this.validValues = validValues;
+ }
+
+ @Override
+ public void validate(String property, Object value) throws ValidationException {
+ for (String valid : validValues) {
+ if (valid.equals(value)) {
+ return;
+ }
+ }
+
+ throw new ValidationException(property, MessageFormat.format("only the following values are allowed: ", validValues.toString()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ImplementationValidator.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ImplementationValidator.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ImplementationValidator.java
new file mode 100755
index 0000000..ad2662e
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ImplementationValidator.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings.validation;
+
+import java.text.MessageFormat;
+
+public class ImplementationValidator implements PropertyValidator {
+
+ public void validate(String property, Object value) throws ValidationException {
+ Class<?> implClass;
+ try {
+ implClass = this.getClass().getClassLoader().loadClass(String.valueOf(value));
+ Object o = implClass.newInstance();
+ o.toString();
+ return;
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ throw new ValidationException(property, MessageFormat.format("Class {0} could not be found!", value));
+ } catch (IllegalAccessException e) {
+ e.printStackTrace();
+ throw new ValidationException(property, MessageFormat.format("Class {0} could not be accessed!", value));
+ } catch (InstantiationException e) {
+ e.printStackTrace();
+ throw new ValidationException(property, MessageFormat.format("Class {0} could not be instantiated!", value));
+ } catch (NoClassDefFoundError e) {
+ e.printStackTrace();
+ throw new ValidationException(property, MessageFormat.format("Class defintiion {0} could not be found!", value));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/NumberPositiveValidator.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/NumberPositiveValidator.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/NumberPositiveValidator.java
new file mode 100755
index 0000000..3c09f07
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/NumberPositiveValidator.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings.validation;
+
+public class NumberPositiveValidator implements PropertyValidator {
+
+ public void validate(String property, Object value) throws ValidationException {
+ if (!(value instanceof Number)) {
+ throw new ValidationException("Only numbers are allowed!");
+ } else {
+ if (value instanceof Long && (long) value < 0) {
+ throw new ValidationException(property, "Only positive values are allowed!");
+ } else if (value instanceof Integer && (int) value < 0) {
+ throw new ValidationException(property, "Only positive values are allowed!");
+ } else if (value instanceof Double && (double) value < 0) {
+ throw new ValidationException(property, "Only positive values are allowed!");
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/PropertyValidator.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/PropertyValidator.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/PropertyValidator.java
new file mode 100755
index 0000000..6acb902
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/PropertyValidator.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings.validation;
+
+public interface PropertyValidator {
+
+ public void validate(String property, Object value) throws ValidationException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/StringNotEmptyValidator.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/StringNotEmptyValidator.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/StringNotEmptyValidator.java
new file mode 100755
index 0000000..c12cf59
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/StringNotEmptyValidator.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings.validation;
+
+import java.text.MessageFormat;
+
+public class StringNotEmptyValidator implements PropertyValidator {
+
+ @Override
+ public void validate(String property, Object value) throws ValidationException {
+ if (value != null && value.toString().trim().isEmpty()) {
+ throw new ValidationException(MessageFormat.format("The property {0} is required and cannot be empty", property));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ValidationException.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ValidationException.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ValidationException.java
new file mode 100755
index 0000000..b560e9f
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/settings/validation/ValidationException.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.settings.validation;
+
+import java.text.MessageFormat;
+
+public class ValidationException extends Exception {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 485240669635915916L;
+ private String property;
+ private String errorCause;
+
+ public ValidationException(String property, String errorMessage) {
+ this.errorCause = errorMessage;
+ this.property = property;
+ }
+
+ public ValidationException(String errorCause) {
+ this.errorCause = errorCause;
+ }
+
+ @Override
+ public String getMessage() {
+ if (property != null && errorCause != null) {
+ return MessageFormat.format("Error setting property {0}, {1}", property, errorCause);
+ } else if (errorCause != null) {
+ return MessageFormat.format("Error setting property value, {0}", errorCause);
+ } else {
+ return "Error setting property value.";
+ }
+ }
+
+ public String getProperty() {
+ return property;
+ }
+
+ public void setProperty(String property) {
+ this.property = property;
+ }
+
+ public String getErrorCause() {
+ return errorCause;
+ }
+
+ public void setErrorCause(String error) {
+ this.errorCause = error;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryService.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryService.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryService.java
new file mode 100755
index 0000000..cc218fb
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryService.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.spark;
+
+import org.apache.spark.sql.SparkSession;
+
+import org.apache.atlas.odf.api.discoveryservice.sync.SyncDiscoveryService;
+
+/**
+ * Interface to be implemented by generic Spark discovery services.
+ *
+ *
+ */
+
+public interface SparkDiscoveryService extends SyncDiscoveryService {
+
+ /**
+ * Sets the Spark context to be used for processing the discovery service.
+ *
+ */
+ void setSparkSession(SparkSession spark);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryServiceBase.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryServiceBase.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryServiceBase.java
new file mode 100755
index 0000000..28c7831
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkDiscoveryServiceBase.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.spark;
+
+import org.apache.spark.sql.SparkSession;
+
+import org.apache.atlas.odf.api.metadata.MetadataStore;
+import org.apache.atlas.odf.api.discoveryservice.SyncDiscoveryServiceBase;
+
+public abstract class SparkDiscoveryServiceBase extends SyncDiscoveryServiceBase implements SparkDiscoveryService {
+ protected SparkSession spark;
+ protected MetadataStore mds;
+
+ @Override
+ public void setSparkSession(SparkSession spark) {
+ this.spark = spark;
+ }
+
+ @Override
+ public void setMetadataStore(MetadataStore mds) {
+ this.mds = mds;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkServiceExecutor.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkServiceExecutor.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkServiceExecutor.java
new file mode 100755
index 0000000..d443303
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkServiceExecutor.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.spark;
+
+import org.apache.atlas.odf.api.discoveryservice.DataSetCheckResult;
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceProperties;
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceRequest;
+import org.apache.atlas.odf.api.discoveryservice.datasets.DataSetContainer;
+import org.apache.atlas.odf.api.discoveryservice.sync.DiscoveryServiceSyncResponse;
+
+/**
+ * Internal interface to be used for processing Spark discovery services.
+ *
+ *
+ */
+
+public interface SparkServiceExecutor {
+
+ DiscoveryServiceSyncResponse runAnalysis(DiscoveryServiceProperties dsri, DiscoveryServiceRequest request);
+
+ DataSetCheckResult checkDataSet(DiscoveryServiceProperties dsri, DataSetContainer dataSetContainer);
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkUtils.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkUtils.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkUtils.java
new file mode 100755
index 0000000..50cb09f
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/spark/SparkUtils.java
@@ -0,0 +1,308 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.spark;
+
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceResult;
+import org.apache.atlas.odf.api.metadata.MetadataStore;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.util.Utils;
+import org.apache.wink.json4j.JSONException;
+import org.apache.wink.json4j.JSONObject;
+
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceSparkEndpoint;
+import org.apache.atlas.odf.api.discoveryservice.datasets.DataSetContainer;
+import org.apache.atlas.odf.api.metadata.MetaDataObjectReference;
+import org.apache.atlas.odf.api.metadata.models.Annotation;
+import org.apache.atlas.odf.api.metadata.models.CachedMetadataStore;
+import org.apache.atlas.odf.api.metadata.models.JDBCConnection;
+import org.apache.atlas.odf.api.metadata.models.JDBCConnectionInfo;
+import org.apache.atlas.odf.api.metadata.models.MetaDataObject;
+import org.apache.atlas.odf.api.metadata.models.Column;
+import org.apache.atlas.odf.api.metadata.models.Connection;
+import org.apache.atlas.odf.api.metadata.models.DataFile;
+import org.apache.atlas.odf.api.metadata.models.RelationalDataSet;
+import org.apache.atlas.odf.api.metadata.models.Table;
+import org.apache.atlas.odf.api.metadata.models.ProfilingAnnotation;
+
+/**
+ * Provides a number of helper methods, mainly for working with Spark data frames.
+ *
+ *
+ */
+
+public class SparkUtils {
+ static Logger logger = Logger.getLogger(SparkUtils.class.getName());
+
+ /**
+ * Creates a Spark data frame from a data set reference stored in a data set container.
+ *
+ * @param sc Current Spark context
+ * @param request dsc Data set container that keeps the reference to the input data set
+ * @return Resulting Spark data frame
+ */
+ public static Dataset<Row> createDataFrame(SparkSession spark, DataSetContainer dsc, MetadataStore mds) {
+ Dataset<Row> df = null;
+ MetaDataObject ds = dsc.getDataSet();
+ if (ds instanceof DataFile) {
+ DataFile dataFile = (DataFile) ds;
+ logger.log(Level.INFO, MessageFormat.format("Reading DataFile {0} from URL {1}.",
+ new Object[] { dataFile.getName(), dataFile.getUrlString() }));
+ df = spark.read().format("csv").option("inferSchema", "true")
+ .option("header", "true").load(dataFile.getUrlString());
+ } else if (ds instanceof Table) {
+ Table table = (Table) ds;
+ MetadataStore availableMetadataStore;
+ if (mds.testConnection() == MetadataStore.ConnectionStatus.OK) {
+ availableMetadataStore = mds;
+ } else if (dsc.getMetaDataCache() != null) {
+ availableMetadataStore = new CachedMetadataStore(dsc.getMetaDataCache());
+ } else {
+ throw new RuntimeException("Discovery service has no access to the metadata store and no metadata cache is available.");
+ }
+ JDBCConnectionInfo connectionInfo = (JDBCConnectionInfo) availableMetadataStore.getConnectionInfo(table);
+ List<Connection> connections = connectionInfo.getConnections();
+ if (connections == null || connections.isEmpty()) {
+ // No connection information is attached to the relational table that was passed to the discovery service.
+ // This is typically caused by the fact that the Spark discovery service cannot access the ODF metadata API in order to retrieve cached objects
+ String msg = "Spark discovery service cannot access the ODF metadata API. Make sure that the ODF REST API is accessible from the discovery service running on the Spark cluster.";
+ logger.log(Level.SEVERE, msg);
+ throw new RuntimeException(msg);
+ }
+ JDBCConnection jdbcConnection = null;
+ for (Connection connection : connections) {
+ if (connection instanceof JDBCConnection) {
+ jdbcConnection = (JDBCConnection) connection;
+ break;
+ }
+ }
+ String driver = null;
+ try {
+ // Get JDBC driver class name needed for populating DataFrame
+ // below
+ driver = DriverManager.getConnection(jdbcConnection.getJdbcConnectionString(), jdbcConnection.getUser(),
+ jdbcConnection.getPassword()).getClass().getName();
+ logger.log(Level.INFO, MessageFormat.format("JDBC driver class name is {0}.", driver));
+ } catch (SQLException e) {
+ String msg = MessageFormat.format("Error connecting to JDBC data source {0}: ",
+ jdbcConnection.getJdbcConnectionString());
+ logger.log(Level.WARNING, msg, e);
+ throw new RuntimeException(msg + Utils.exceptionString(e));
+ }
+ String schemaName = connectionInfo.getSchemaName();
+ String url = jdbcConnection.getJdbcConnectionString() + ":currentSchema=" + schemaName + ";user="
+ + jdbcConnection.getUser() + ";password=" + jdbcConnection.getPassword() + ";";
+ String dbtable = schemaName + "." + table.getName();
+ String msg = "Using JDBC parameters url: {0}, dbtable: {1}, driver: {2} to connect to DB2 database.";
+ logger.log(Level.INFO, MessageFormat.format(msg, new Object[] { url, dbtable, driver }));
+ Map<String, String> options = new HashMap<String, String>();
+ options.put("url", url);
+ options.put("dbtable", dbtable);
+ options.put("driver", "com.ibm.db2.jcc.DB2Driver");
+ df = spark.read().format("jdbc").options(options).load();
+ }
+ return df;
+ }
+
+ /**
+ * Generates ODF annotations from a annotation data frames.
+ *
+ * @param container Data set container that contains the reference to the data set to be annotated
+ * @param annotationDataFrameMap Maps the annotation types to be created with the annotation data frames that contain the actual annotation data
+ * @return Result object that contains a list of ODF annotations
+ */
+ public static DiscoveryServiceResult createAnnotationsFromDataFrameMap(DataSetContainer container, Map<String, Dataset<Row>> annotationDataFrameMap, MetadataStore mds) throws RuntimeException {
+ RelationalDataSet tab = (RelationalDataSet) container.getDataSet();
+ DiscoveryServiceResult result = new DiscoveryServiceResult();
+
+ // Map input table columns to metadata object references
+ Map<String, MetaDataObjectReference> columnReferencesByName = new HashMap<>();
+
+ List<Column> colList ;
+ if (mds.testConnection() == MetadataStore.ConnectionStatus.OK) {
+ colList = mds.getColumns(tab);
+ } else if (container.getMetaDataCache() != null) {
+ CachedMetadataStore cacheReader = new CachedMetadataStore(container.getMetaDataCache());
+ colList = cacheReader.getColumns(tab);
+ } else {
+ throw new RuntimeException("Discovery service has no access to the metadata store and no metadata cache is available.");
+ }
+
+ for (MetaDataObject colMDO : colList) {
+ Column oMColumn = (Column) colMDO;
+ columnReferencesByName.put(oMColumn.getName(), oMColumn.getReference());
+ }
+
+ List<Annotation> annotations = new ArrayList<>();
+ Dataset<Row> df = null;
+ for (Map.Entry<String, Dataset<Row>> entry : annotationDataFrameMap.entrySet()) {
+ String annotationType = entry.getKey();
+ df = entry.getValue();
+ String columnToBeAnnotated = null;
+ int rowNumber = 0;
+ try {
+ List<Row> rows = df.collectAsList();
+ String[] columnNames = df.columns();
+ StructType st = df.schema();
+
+ for (rowNumber = 0; rowNumber < rows.size(); rowNumber++) {
+ if (columnNames[0].equals(DiscoveryServiceSparkEndpoint.ANNOTATION_PROPERTY_COLUMN_NAME)) {
+ // Generate column annotations by mapping DataFrame
+ // table column values to annotation properties
+ // Column ANNOTATION_PROPERTY_COLUMN_NAME represents the
+ // column to be annotated
+ columnToBeAnnotated = rows.get(rowNumber).getString(0);
+ MetaDataObjectReference annotatedColumn = columnReferencesByName.get(columnToBeAnnotated);
+ if (annotatedColumn != null) {
+ logger.log(Level.FINE, MessageFormat.format("Annotating column {0}:", columnToBeAnnotated));
+ annotations.add((Annotation) getAnnotation(st, columnNames, annotationType, rows.get(rowNumber),
+ annotatedColumn));
+ } else {
+ logger.log(Level.FINE, "Column " + columnToBeAnnotated
+ + " returned by the Spark service does not match any column of the input data set.");
+ }
+ } else {
+ // Creating table annotations
+ logger.log(Level.INFO,
+ MessageFormat.format(
+ "Data frame does not contain column {0}. Creating table annotations.",
+ DiscoveryServiceSparkEndpoint.ANNOTATION_PROPERTY_COLUMN_NAME));
+ annotations.add((Annotation) getAnnotation(st, columnNames, annotationType, rows.get(rowNumber),
+ container.getDataSet().getReference()));
+ }
+ }
+ } catch (JSONException exc) {
+ String msg = MessageFormat.format(
+ "Error processing results returned by DataFrame row {0} column {1}. See ODF application lof for details.",
+ new Object[] { rowNumber, columnToBeAnnotated });
+ logger.log(Level.WARNING, msg);
+ throw new RuntimeException(msg, exc);
+ }
+ }
+ result.setAnnotations(annotations);
+ return result;
+ }
+
+ /**
+ * Creates a single ODF annotation from a row of input data.
+ *
+ * @param st Data types of the annotation attributes
+ * @param columnNames Names of the annotation attributes
+ * @param row Input data that represents the values of the annotation attributes
+ * @return A single ODF annotation object
+ */
+ public static Annotation getAnnotation(StructType st, String[] columnNames, String annotationType, Row row,
+ MetaDataObjectReference annotatedObject) throws JSONException {
+ ProfilingAnnotation an = new ProfilingAnnotation();
+ an.setAnnotationType(annotationType);
+ an.setProfiledObject(annotatedObject);
+ JSONObject jsonProperties = new JSONObject();
+ for (int j = 0; j < columnNames.length; j++) {
+ if (!columnNames[j].equals(DiscoveryServiceSparkEndpoint.ANNOTATION_PROPERTY_COLUMN_NAME)) {
+ if (columnNames[j].equals(DiscoveryServiceSparkEndpoint.ANNOTATION_SUMMARY_COLUMN_NAME)) {
+ an.setSummary(row.getString(j));
+ } else {
+ String annotationPropertyName = columnNames[j];
+ DataType dataType = st.apply(annotationPropertyName).dataType();
+ if (dataType == DataTypes.IntegerType) {
+ jsonProperties.put(annotationPropertyName, row.getInt(j));
+ } else if (dataType == DataTypes.DoubleType) {
+ jsonProperties.put(annotationPropertyName, row.getDouble(j));
+ } else if (dataType == DataTypes.BooleanType) {
+ jsonProperties.put(annotationPropertyName, row.getBoolean(j));
+ } else if (dataType == DataTypes.FloatType) {
+ jsonProperties.put(annotationPropertyName, row.getFloat(j));
+ } else if (dataType == DataTypes.LongType) {
+ jsonProperties.put(annotationPropertyName, row.getLong(j));
+ } else if (dataType == DataTypes.ShortType) {
+ jsonProperties.put(annotationPropertyName, row.getShort(j));
+ } else {
+ // Return all other data types as String
+ jsonProperties.put(annotationPropertyName, row.getString(j));
+ }
+ logger.log(Level.FINE, "Set attribute " + annotationPropertyName + " to value " + row.get(j) + ".");
+ }
+ }
+ }
+ an.setJsonProperties(jsonProperties.toString());
+ return an;
+ }
+
+ /**
+ * Transposes a Spark data frame by replacing its rows by its columns. All input columns are expected to be of type Double.
+ * The fist column of the resulting data frame contains the column names of the input data frame and is of data type String. All other output columns are of type Double.
+ *
+ * @param sc Current Spark context
+ * @param origDataFrame Data frame to be transposed
+ * @return Transposed data frame
+ */
+ public static Dataset<Row> transposeDataFrame(SparkSession spark, Dataset<Row> origDataFrame) {
+ Dataset<Row> transposedDataFrame = null;
+ String[] origColumnNames = origDataFrame.columns();
+ int origNumberColumns = origColumnNames.length;
+ List<Row> origRows = origDataFrame.collectAsList();
+ int origNumberRows = origRows.size();
+ List<Row> transposedRows = new ArrayList<Row>();
+
+ // Loop through columns of original DataFrame
+ for (int i = 1; i < origNumberColumns; i++) {
+ Object[] transposedRow = new Object[origNumberRows + 1];
+ transposedRow[0] = origColumnNames[i];
+ // Loop trough rows of original DataFrame
+ for (int j = 0; j < origNumberRows; j++) {
+ if (origRows.get(j).getString(i) == null) {
+ transposedRow[j + 1] = null;
+ } else {
+ try {
+ transposedRow[j + 1] = Double.parseDouble(origRows.get(j).getString(i));
+ } catch(NumberFormatException e) {
+ if (logger.getLevel() == Level.FINEST) {
+ String msg = MessageFormat.format("Cannot convert DataFrame column {0} row {1} value ''{2}'' to Double.", new Object[] { i, j, origRows.get(j).getString(i) });
+ logger.log(Level.FINEST, msg);
+ }
+ // Return null for all non-numeric fields
+ transposedRow[j + 1] = null;
+ }
+ }
+ }
+ transposedRows.add(RowFactory.create(transposedRow));
+ }
+
+ // Store original column name in first column of transposed DataFrame
+ StructField[] transposedColumnNames = new StructField[origNumberRows + 1];
+ transposedColumnNames[0] = DataTypes.createStructField(origColumnNames[0], DataTypes.StringType, false);
+ for (int j = 0; j < origNumberRows; j++) {
+ transposedColumnNames[j + 1] = DataTypes.createStructField(origRows.get(j).getString(0), DataTypes.DoubleType, false);
+ }
+ StructType st = DataTypes.createStructType(transposedColumnNames);
+ transposedDataFrame = spark.createDataFrame(transposedRows, st);
+ return transposedDataFrame;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-api/src/main/java/org/apache/atlas/odf/api/utils/ODFLogConfig.java
----------------------------------------------------------------------
diff --git a/odf/odf-api/src/main/java/org/apache/atlas/odf/api/utils/ODFLogConfig.java b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/utils/ODFLogConfig.java
new file mode 100755
index 0000000..e4fcaa3
--- /dev/null
+++ b/odf/odf-api/src/main/java/org/apache/atlas/odf/api/utils/ODFLogConfig.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.api.utils;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.logging.FileHandler;
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.logging.SimpleFormatter;
+
+/**
+ * Class to be used for log configuration.
+ * It reads the system property odf.logspec which value must be of the form
+ * <Level>,<FilePattern>
+ * For instance
+ * ALL,/tmp/odf-trace.log
+ *
+ *
+ */
+public class ODFLogConfig {
+
+ void log(String s) {
+ System.out.println(s);
+ }
+
+ public static class ODFFileHandler extends FileHandler {
+ public ODFFileHandler(String pattern) throws IOException {
+ super(pattern);
+ }
+ }
+
+ Handler createHandler(String odfLogFilePattern) {
+ Handler result = null;
+ Formatter formatter = new SimpleFormatter();
+ try {
+ // 1MB file limit, 3 files
+ result = new ODFFileHandler(odfLogFilePattern);
+ result.setFormatter(formatter);
+ } catch (Exception exc) {
+ exc.printStackTrace();
+ return null;
+ }
+ return result;
+ }
+
+ private ODFLogConfig() {
+ try {
+ String logSpec = System.getProperty("odf.logspec");
+ log("ODF Logging spec of system property odf.logspec: " + logSpec);
+ if (logSpec == null) {
+ logSpec = System.getenv("ODFLOGSPEC");
+ log("ODF Logging spec of env var ODFLOGSPEC " + logSpec);
+ if (logSpec == null) {
+ return;
+ }
+ }
+ int ix = logSpec.indexOf(",");
+ if (ix == -1) {
+ return;
+ }
+ String levelString = logSpec.substring(0, ix);
+
+ String odfLogFilePattern = logSpec.substring(ix + 1);
+ String msg = MessageFormat.format("Configuring ODF logging with level {0} and log file: {1}", new Object[] { levelString, odfLogFilePattern });
+ log(msg);
+
+ Handler odfHandler = createHandler(odfLogFilePattern);
+ if (odfHandler == null) {
+ return;
+ }
+ Level level = Level.parse(levelString);
+ Logger odfRootLogger = Logger.getLogger("org.apache.atlas.odf");
+
+ // remove existing handler
+ for (Handler h : odfRootLogger.getHandlers()) {
+ if (h instanceof ODFFileHandler) {
+ odfRootLogger.removeHandler(h);
+ }
+ }
+
+ odfRootLogger.setLevel(level);
+ odfHandler.setLevel(level);
+ odfRootLogger.addHandler(odfHandler);
+ log("ODF logger configured.");
+ } catch (Exception exc) {
+ exc.printStackTrace();
+ }
+ }
+
+ static Object lockObject = new Object();
+ static ODFLogConfig config = null;
+
+ public static void run() {
+ synchronized (lockObject) {
+ if (config == null) {
+ config = new ODFLogConfig();
+ }
+ }
+ }
+}