You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/06/28 05:57:22 UTC
[09/25] incubator-atlas git commit: ATLAS-1898: initial commit of ODF
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/MockSparkServiceExecutor.java
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/MockSparkServiceExecutor.java b/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/MockSparkServiceExecutor.java
new file mode 100755
index 0000000..30848bd
--- /dev/null
+++ b/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/MockSparkServiceExecutor.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.core.test.spark;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.atlas.odf.api.discoveryservice.DataSetCheckResult;
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceRequest;
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceResponse;
+import org.apache.atlas.odf.api.spark.SparkServiceExecutor;
+import org.apache.atlas.odf.json.JSONUtils;
+import org.apache.wink.json4j.JSONException;
+
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceProperties;
+import org.apache.atlas.odf.api.discoveryservice.DiscoveryServiceSparkEndpoint;
+import org.apache.atlas.odf.api.discoveryservice.datasets.DataSetContainer;
+import org.apache.atlas.odf.api.discoveryservice.sync.DiscoveryServiceSyncResponse;
+
+public class MockSparkServiceExecutor implements SparkServiceExecutor {
+ Logger logger = Logger.getLogger(MockSparkServiceExecutor.class.getName());
+
+ public DataSetCheckResult checkDataSet(DiscoveryServiceProperties dsri, DataSetContainer dataSetContainer) {
+ DataSetCheckResult checkResult = new DataSetCheckResult();
+ checkResult.setDataAccess(DataSetCheckResult.DataAccess.Possible);
+ return checkResult;
+ }
+
+ @Override
+ public DiscoveryServiceSyncResponse runAnalysis(DiscoveryServiceProperties dsri, DiscoveryServiceRequest request) {
+ logger.log(Level.INFO, "Starting Spark mock application.");
+ DiscoveryServiceSparkEndpoint sparkEndpoint;
+ try {
+ sparkEndpoint = JSONUtils.convert(dsri.getEndpoint(), DiscoveryServiceSparkEndpoint.class);
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ if (sparkEndpoint.getJar() == null) {
+ throw new RuntimeException("Spark application is not set in Spark endpoint.");
+ }
+ logger.log(Level.INFO, "Application name is {0}.", sparkEndpoint.getJar());
+ logger.log(Level.INFO, "Spark application finished.");
+ DiscoveryServiceSyncResponse response = new DiscoveryServiceSyncResponse();
+ response.setCode(DiscoveryServiceResponse.ResponseCode.OK);
+ response.setDetails("Discovery service completed successfully.");
+ return response;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/SimpleSparkDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/SimpleSparkDiscoveryServiceTest.java b/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/SimpleSparkDiscoveryServiceTest.java
new file mode 100755
index 0000000..661cfe2
--- /dev/null
+++ b/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/spark/SimpleSparkDiscoveryServiceTest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.core.test.spark;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.logging.Level;
+
+import org.apache.atlas.odf.api.analysis.AnalysisRequest;
+import org.apache.atlas.odf.api.analysis.AnalysisRequestStatus;
+import org.apache.atlas.odf.api.analysis.AnalysisResponse;
+import org.apache.atlas.odf.api.metadata.MetadataStore;
+import org.apache.atlas.odf.api.metadata.models.DataFile;
+import org.apache.atlas.odf.api.metadata.models.RelationalDataSet;
+import org.apache.atlas.odf.core.metadata.DefaultMetadataStore;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.atlas.odf.api.metadata.MetaDataObjectReference;
+import org.apache.atlas.odf.api.ODFFactory;
+import org.apache.atlas.odf.api.analysis.AnalysisManager;
+import org.apache.atlas.odf.core.test.ODFTestBase;
+
+public class SimpleSparkDiscoveryServiceTest extends ODFTestBase {
+
+ public static int WAIT_MS_BETWEEN_POLLING = 500;
+ public static int MAX_NUMBER_OF_POLLS = 500;
+
+ @Test
+ public void testSparkService() throws Exception{
+ log.info("Running request ");
+ AnalysisManager analysisManager = new ODFFactory().create().getAnalysisManager();
+ AnalysisRequest request = new AnalysisRequest();
+ List<MetaDataObjectReference> dataSetRefs = new ArrayList<>();
+ MetadataStore mds = new ODFFactory().create().getMetadataStore();
+ if (!(mds instanceof DefaultMetadataStore)) {
+ throw new RuntimeException(MessageFormat.format("This tests does not work with metadata store implementation \"{0}\" but only with the DefaultMetadataStore.", mds.getClass().getName()));
+ }
+ DefaultMetadataStore defaultMds = (DefaultMetadataStore) mds;
+ defaultMds.resetAllData();
+ RelationalDataSet dataSet = new DataFile();
+ MetaDataObjectReference ref = new MetaDataObjectReference();
+ ref.setId("datafile-mock");
+ dataSet.setReference(ref);
+ defaultMds.createObject(dataSet);
+ defaultMds.commit();
+ dataSetRefs.add(dataSet.getReference());
+ request.setDataSets(dataSetRefs);
+ List<String> serviceIds = Arrays.asList(new String[]{"spark-service-test"});
+ request.setDiscoveryServiceSequence(serviceIds);
+
+ log.info("Starting analyis");
+ AnalysisResponse response = analysisManager.runAnalysis(request);
+ Assert.assertNotNull(response);
+ String requestId = response.getId();
+ Assert.assertNotNull(requestId);
+ log.info("Request id is " + requestId + ".");
+
+ log.info("Waiting for request to finish");
+ AnalysisRequestStatus status = null;
+ int maxPolls = MAX_NUMBER_OF_POLLS;
+ do {
+ status = analysisManager.getAnalysisRequestStatus(requestId);
+ log.log(Level.INFO, "Poll request for request ID ''{0}'', state: ''{1}'', details: ''{2}''", new Object[] { requestId, status.getState(), status.getDetails() });
+ maxPolls--;
+ try {
+ Thread.sleep(WAIT_MS_BETWEEN_POLLING);
+ } catch (InterruptedException e) {
+ log.log(Level.INFO, "Exception thrown: ", e);
+ }
+ } while (maxPolls > 0 && (status.getState() == AnalysisRequestStatus.State.ACTIVE || status.getState() == AnalysisRequestStatus.State.QUEUED || status.getState() == AnalysisRequestStatus.State.NOT_FOUND));
+ if (maxPolls == 0) {
+ log.log(Level.INFO, "Request ''{0}'' is not finished yet, don't wait for it", requestId);
+ }
+ Assert.assertEquals(AnalysisRequestStatus.State.FINISHED, status.getState());
+ log.log(Level.INFO, "Request ''{0}'' is finished.", requestId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/store/MockConfigurationStorage.java
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/store/MockConfigurationStorage.java b/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/store/MockConfigurationStorage.java
new file mode 100755
index 0000000..191d337
--- /dev/null
+++ b/odf/odf-core/src/test/java/org/apache/atlas/odf/core/test/store/MockConfigurationStorage.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.odf.core.test.store;
+
+import org.apache.wink.json4j.JSONException;
+import org.apache.wink.json4j.JSONObject;
+
+import org.apache.atlas.odf.core.configuration.ConfigContainer;
+import org.apache.atlas.odf.core.store.ODFConfigurationStorage;
+import org.apache.atlas.odf.json.JSONUtils;
+
+public class MockConfigurationStorage implements ODFConfigurationStorage {
+
+ static JSONObject config;
+
+ static {
+ try {
+ config = new JSONObject(MockConfigurationStorage.class.getClassLoader().getResourceAsStream("org/apache/atlas/odf/core/test/internal/odf-initial-configuration.json"));
+ } catch (JSONException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void storeConfig(ConfigContainer container) {
+ try {
+ config = JSONUtils.toJSONObject(container);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public ConfigContainer getConfig(ConfigContainer defaultConfig) {
+ try {
+ return JSONUtils.fromJSON(config.write(), ConfigContainer.class);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void onConfigChange(ConfigContainer container) {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void addPendingConfigChange(String changeId) {
+ // do nothing
+ }
+
+ @Override
+ public void removePendingConfigChange(String changeId) {
+ // do nothing
+ }
+
+ @Override
+ public boolean isConfigChangePending(String changeId) {
+ return false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/resources/META-INF/odf/odf-runtimes.txt
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/resources/META-INF/odf/odf-runtimes.txt b/odf/odf-core/src/test/resources/META-INF/odf/odf-runtimes.txt
new file mode 100755
index 0000000..d421138
--- /dev/null
+++ b/odf/odf-core/src/test/resources/META-INF/odf/odf-runtimes.txt
@@ -0,0 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+TestServiceRuntime
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/integrationtest/metadata/internal/atlas/nested_annotation_example.json
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/integrationtest/metadata/internal/atlas/nested_annotation_example.json b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/integrationtest/metadata/internal/atlas/nested_annotation_example.json
new file mode 100755
index 0000000..34dbf78
--- /dev/null
+++ b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/integrationtest/metadata/internal/atlas/nested_annotation_example.json
@@ -0,0 +1,111 @@
+{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "obj1":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ },
+ "arr1":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ }
+ ],
+ "obj2":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nobj21":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnarr211":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ }
+ ]
+ },
+ "narr21":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnarr211":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ }
+ ]
+ }
+ ]
+ },
+ "obj3":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nobj31":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnobj31":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ }
+ },
+ "narr31":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnarr311":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnnarr3111":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ },
+ "obj4":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nobj41":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nobj411":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnnarr4111":[
+ {
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999,
+ "nnobj41111":{
+ "prop1":"mystring",
+ "prop2":999,
+ "prop3":999.999
+ }
+ }
+ ]
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/annotation/annotexttest1.json
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/annotation/annotexttest1.json b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/annotation/annotexttest1.json
new file mode 100755
index 0000000..146748d
--- /dev/null
+++ b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/annotation/annotexttest1.json
@@ -0,0 +1,8 @@
+{
+ "javaClass": "aHopefullyUnknownClass",
+ "profiledObject": null,
+ "annotationType": "MySubType",
+ "analysisRun": "bla",
+ "newProp1": "newProp1Value",
+ "newProp2": 4237
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/internal/odf-initial-configuration.json
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/internal/odf-initial-configuration.json b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/internal/odf-initial-configuration.json
new file mode 100755
index 0000000..9757e51
--- /dev/null
+++ b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/internal/odf-initial-configuration.json
@@ -0,0 +1,114 @@
+{
+ "odf" : {
+ "instanceId" : "odf-default-id-CHANGEME",
+ "odfUrl" : "https://localhost:58081/odf-web-1.2.0-SNAPSHOT",
+ "odfUser" : "odf",
+ "odfPassword" : "ZzTeX3hKtVORgks+2TaLPWxerucPBoxK",
+ "runNewServicesOnRegistration": false,
+ "runAnalysisOnImport": false,
+ "reuseRequests": true,
+ "discoveryServiceWatcherWaitMs": 2000,
+ "enableAnnotationPropagation": true,
+ "messagingConfiguration": {
+ "type": "com.ibm.iis.odf.api.settings.KafkaMessagingConfiguration",
+ "analysisRequestRetentionMs": 86400000,
+ "queueConsumerWaitMs": 2000,
+ "kafkaBrokerTopicReplication": 1,
+ "kafkaConsumerConfig": {
+ "offsetsStorage": "kafka",
+ "zookeeperSessionTimeoutMs": 400,
+ "zookeeperConnectionTimeoutMs": 6000
+ }
+ },
+ "userDefined": {
+ }
+ },
+ "registeredServices": [{
+ "id": "asynctestservice",
+ "name": "Async test",
+ "description": "The async test service",
+ "resultingAnnotationTypes": [
+ "AsyncTestDummyAnnotation"
+ ],
+ "endpoint": {
+ "runtimeName": "Java",
+ "className": "com.ibm.iis.odf.core.test.discoveryservice.TestAsyncDiscoveryService1"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "asynctestservice-with-annotations",
+ "name": "Async test including metadata access",
+ "description": "The async test service writing annotations",
+ "endpoint": {
+ "runtimeName": "Java",
+ "className": "com.ibm.iis.odf.core.test.discoveryservice.TestAsyncDiscoveryServiceWritingAnnotations1"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "synctestservice",
+ "name": "Sync test",
+ "description": "The Sync test service",
+ "resultingAnnotationTypes": [
+ "SyncTestDummyAnnotation"
+ ],
+ "endpoint": {
+ "runtimeName": "Java",
+ "className": "com.ibm.iis.odf.core.test.discoveryservice.TestSyncDiscoveryService1"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "synctestservice-with-annotations",
+ "name": "Sync test with annotations",
+ "description": "The Sync test service writing annotations",
+ "endpoint": {
+ "runtimeName": "Java",
+ "className": "com.ibm.iis.odf.core.test.discoveryservice.TestSyncDiscoveryServiceWritingAnnotations1"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "synctestservice-with-extendedannotations",
+ "name": "Sync test with extended annotations",
+ "description": "The Sync test service writing annotations with extension mechanism",
+ "endpoint": {
+ "runtimeName": "Java",
+ "className": "com.ibm.iis.odf.core.test.annotation.TestSyncDiscoveryServiceWritingExtendedAnnotations"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "synctestservice-with-json-annotations",
+ "name": "Sync test with json annotations",
+ "description": "The Sync test service writing annotations returned from a json file",
+ "endpoint": {
+ "runtimeName": "Java",
+ "className": "com.ibm.iis.odf.core.test.annotation.TestSyncDiscoveryServiceWritingJsonAnnotations"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "spark-service-test",
+ "name": "Simple Spark mock test",
+ "description": "The Spark test is calling a mock version of the SparkAppExecutor",
+ "endpoint": {
+ "runtimeName": "Spark",
+ "inputMethod": "DataFrame",
+ "jar": "my-example-application-jar",
+ "className": "my-example-class-name"
+ },
+ "parallelismCount" : 2
+ },
+ {
+ "id": "testruntimeservice",
+ "name": "Runtime test service",
+ "description": "Runtime test service description",
+ "endpoint": {
+ "runtimeName": "TestServiceRuntime"
+ }
+ }
+
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/messaging/kafka/tracker1.json
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/messaging/kafka/tracker1.json b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/messaging/kafka/tracker1.json
new file mode 100755
index 0000000..b884aca
--- /dev/null
+++ b/odf/odf-core/src/test/resources/org/apache/atlas/odf/core/test/messaging/kafka/tracker1.json
@@ -0,0 +1,31 @@
+{
+ "user": "isadmin",
+ "lastModified": "1443795291000",
+ "discoveryServiceRequests": [{
+ "dataSetContainer": {
+ "dataSet": {
+ "javaClass": "com.ibm.iis.odf.core.metadata.models.Document",
+ "name": "someDocument",
+ "reference": {
+ "id": "testdataset"
+ }
+ }
+ },
+ "discoveryServiceId": "testservice"
+ }],
+ "nextDiscoveryServiceRequest": 1,
+ "request": {
+ "dataSets": [{
+ "id": "testdataset"
+ }],
+ "id": "testid"
+ },
+ "status": "FINISHED",
+ "statusDetails": "All discovery services run successfully",
+ "discoveryServiceResponses": [{
+ "type": "async",
+ "runId": "IARUNID6f49fdfd-89ce-4d46-9067-b3a4db4698ba",
+ "details": "IA has run successfully",
+ "code": "OK"
+ }]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-core/src/test/resources/org/apache/atlas/odf/odf-implementation.properties
----------------------------------------------------------------------
diff --git a/odf/odf-core/src/test/resources/org/apache/atlas/odf/odf-implementation.properties b/odf/odf-core/src/test/resources/org/apache/atlas/odf/odf-implementation.properties
new file mode 100755
index 0000000..18109c4
--- /dev/null
+++ b/odf/odf-core/src/test/resources/org/apache/atlas/odf/odf-implementation.properties
@@ -0,0 +1,20 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+## USE for TESTs only
+
+
+ODFConfigurationStorage=MockConfigurationStorage
+DiscoveryServiceQueueManager=MockQueueManager
+SparkServiceExecutor=MockSparkServiceExecutor
+NotificationManager=TestNotificationManager
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/.gitignore
----------------------------------------------------------------------
diff --git a/odf/odf-doc/.gitignore b/odf/odf-doc/.gitignore
new file mode 100755
index 0000000..8b22f9d
--- /dev/null
+++ b/odf/odf-doc/.gitignore
@@ -0,0 +1,19 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+target
+.settings
+.classpath
+.project
+.factorypath
+.DS_Store
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/README.txt
----------------------------------------------------------------------
diff --git a/odf/odf-doc/README.txt b/odf/odf-doc/README.txt
new file mode 100755
index 0000000..80dbb61
--- /dev/null
+++ b/odf/odf-doc/README.txt
@@ -0,0 +1,3 @@
+The documentation project is based on the Maven Site Plugin and Maven Doxia. The resulting war file is merged into the war file of the sdp-web project using the overlay mechanism of the Maven War Plugin. The resulting documentation is available through the getting started page of the SDP web console.
+
+Edit the src/site/markdown/*.md files in order to update the documentation. The structure of the web site can be changed in file src/site/site.xml.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/pom.xml
----------------------------------------------------------------------
diff --git a/odf/odf-doc/pom.xml b/odf/odf-doc/pom.xml
new file mode 100755
index 0000000..6ebffcf
--- /dev/null
+++ b/odf/odf-doc/pom.xml
@@ -0,0 +1,163 @@
+<?xml version="1.0"?>
+<!--
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+ xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.atlas.odf</groupId>
+ <artifactId>odf</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>odf-doc</artifactId>
+ <packaging>war</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ <version>1.1.1</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.atlas.odf</groupId>
+ <artifactId>odf-api</artifactId>
+ <version>1.2.0-SNAPSHOT</version>
+ <scope>compile</scope>
+ </dependency>
+ <!-- The following dependencies are required by Spark Discovery Services only and are provided by the Spark cluster -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_2.11</artifactId>
+ <version>2.1.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_2.11</artifactId>
+ <version>2.1.0</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-war-plugin</artifactId>
+ <version>2.6</version>
+ <configuration>
+ <webResources>
+ <resource>
+ <directory>${project.build.directory}/site</directory>
+ <targetPath>/doc</targetPath>
+ </resource>
+ </webResources>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.10.3</version>
+ <configuration>
+ <sourcepath>${basedir}/../odf-api/src/main/java</sourcepath>
+ <outputDirectory>${project.build.directory}/doc</outputDirectory>
+ <excludePackageNames>org.apache.atlas.odf.core.metadata.atlas:org.apache.atlas.odf.core.metadata.importer:org.apache.atlas.odf.core.metadata.internal:org.apache.atlas.odf.json</excludePackageNames>
+ </configuration>
+ <executions>
+ <execution>
+ <id>generate-javadocs</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>javadoc</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <port>9000</port>
+ <tempWebappDirectory>${basedir}/target/site/tempdir</tempWebappDirectory>
+ <generateProjectInfo>false</generateProjectInfo>
+ <generateReports>false</generateReports>
+ <inputEncoding>UTF-8</inputEncoding>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ <executions>
+ <execution>
+ <id>generate-html</id>
+ <phase>validate</phase>
+ <goals>
+ <goal>site</goal>
+ </goals>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>1.3</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <!-- this section compiles the tutorial project to check if the code is valid.
+ -->
+ <!--
+ <plugin>
+ <artifactId>maven-invoker-plugin</artifactId>
+ <version>2.0.0</version>
+ <configuration>
+ <projectsDirectory>src/site/resources/tutorial-projects</projectsDirectory>
+ <cloneProjectsTo>${project.build.directory}/tutorial-projects-build</cloneProjectsTo>
+ </configuration>
+ <executions>
+ <execution>
+ <id>compile-tutorial-projects</id>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ -->
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <!-- Turn off additional checks for maven-javadoc-plugin that will cause build errors when using Java 8 -->
+ <!-- See http://stackoverflow.com/questions/22528767/how-to-work-around-the-stricter-java-8-javadoc-when-using-maven -->
+ <id>disable-java8-doclint</id>
+ <activation>
+ <jdk>[1.8,)</jdk>
+ </activation>
+ <properties>
+ <additionalparam>-Xdoclint:none</additionalparam>
+ </properties>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/main/webapp/WEB-INF/web.xml b/odf/odf-doc/src/main/webapp/WEB-INF/web.xml
new file mode 100755
index 0000000..104b5e4
--- /dev/null
+++ b/odf/odf-doc/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,19 @@
+<!--
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+<!DOCTYPE web-app PUBLIC
+ "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
+ "http://java.sun.com/dtd/web-app_2_3.dtd" >
+<web-app>
+ <display-name>odf-doc</display-name>
+</web-app>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/api-reference.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/api-reference.md b/odf/odf-doc/src/site/markdown/api-reference.md
new file mode 100755
index 0000000..f0bb24f
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/api-reference.md
@@ -0,0 +1,19 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# API reference
+
+[General ODF API reference](../swagger).
+
+[Java Docs for ODF services](./apidocs/index.html)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/build.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/build.md b/odf/odf-doc/src/site/markdown/build.md
new file mode 100755
index 0000000..4a6d7ac
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/build.md
@@ -0,0 +1,113 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Build
+
+This page describes how to build ODF.
+
+## Prerequisites
+
+You need git, Maven, and Python (2.7 (not 3!)) available on the command line.
+If you run these commands and you see similar output you should be all set:
+
+ $ mvn -v
+ Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-10T17:41:47+01:00)
+ ...
+
+ $ python -V
+ Python 2.7.10
+
+ $ git --version
+ git version 2.7.4
+
+The build currently depends on the [IA data quality project][3] (https://github.com/YSAILLET/dataquality) to be available. This can be achieved in two ways:
+1. Make sure that you can reach the [IIS Maven Repository][1]. If this doesn't work, try to authenticate against the Littleton firewall, e.g., by opening the [RTC dashboard][2].
+2. Build the [IA data quality project][3] on your local machine, typically with these commands
+
+ git clone https://github.com/dataquality
+ cd dataquality
+ mvn clean install
+
+
+### Additional Prerequisites on Windows
+
+- For the build: The directory C:\tmp needs to exist
+- For the tests and the test environment to run properly: The `HADOOP_HOME` environment variable must be set to a location where the Hadoop [winutils.exe](http://public-repo-1.hortonworks.com/hdp-win-alpha/winutils.exe) file is available in a the bin folder. For example, if the environment variable is set to `HADOOP_HOME=c:\hadoop`, the file needs to be available at `c:\hadoop\bin\winutils.exe`.
+- In your Maven install directory go to bin and copy mvn.cmd to mvn.bat
+
+## Building
+
+To build, clone the repository and perform a maven build in the toplevel directory. These commands should do the trick:
+
+ git clone https://github.com/Analytics/open-discovery-framework.git
+ cd open-discovery-framework
+ mvn clean install
+
+Add the `-Dreduced-build` option to build and test only the core components and services of ODF:
+
+ mvn clean install -Dreduced-build
+
+## Fast build without tests or with reduced tests
+
+To build without running tests run maven with the following options (The second one prevents the test Atlas instance from being started and stopped):
+
+ mvn clean install -DskipTests -Duse.running.atlas
+
+Use the `-Dreduced-tests` option to run only a reduced set of tests:
+
+ mvn clean install -Dreduced-tests
+
+This will skip all integration tests (i.e. all tests that involve Atlas) and also some of the long running tests. The option may be combined with the `-Dreduced-build` option introduced above.
+
+## Building the test environment
+
+You can build a test environment that contains Atlas and
+Kafka, and Jetty by running these commands:
+
+ cd odf-test-env
+ mvn package
+
+This will create a zip file with the standalone test environment under
+``odf-test-env/target/odf-test-env-0.1.0-SNAPSHOT-bin.zip``.
+See the contents of this zip file or the [documentation section on the test environment](test-env.html)
+for details.
+
+Congrats! You have just built ODF.
+This should be enough to get you going. See below for additional information
+on different aspects of the build.
+
+## Additional Information
+
+### Working with Eclipse
+
+To build with Eclipse you must have the maven m2e plugin and EGit installed (e.g., search for "m2e maven integration for eclipse" and "egit", respectively, on the Eclipse marketplace).
+
+- Clone the repository into some directory as above, e.g., /home/code/odf.
+- Open Eclipse with a workspace in a different directory.
+- Go to File -> Import -> Maven -> Existing Maven projects.
+- Enter /home/code/odf as the root directory.
+- Select all projects and click Finish.
+- Internally, Eclipse will now perform Maven builds but you can work with the code as usual.
+
+If you want to build via Run configurations be aware that this will not work with the embedded
+maven provided by the m2e plugin. Instead you will have to do this:
+
+- Open Windows -> Preferences -> Maven -> Installations
+- Add a new installation pointing to your external Maven installation
+- For each run configuration you use, select the new installation in the Maven runtime dropdown
+(you might also have to set JAVA_HOME in the environment tab).
+
+ [1]: http://iis-repo.swg.usma.ibm.com:8080/archiva/repository/all/
+ [2]: https://ips-rtc.swg.usma.ibm.com/jazz/web/projects
+ [3]: https://github.ibm.com/YSAILLET/dataquality
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/configuration.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/configuration.md b/odf/odf-doc/src/site/markdown/configuration.md
new file mode 100755
index 0000000..c4d0c72
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/configuration.md
@@ -0,0 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Configuration
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/data-model.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/data-model.md b/odf/odf-doc/src/site/markdown/data-model.md
new file mode 100755
index 0000000..5905799
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/data-model.md
@@ -0,0 +1,120 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Data Model
+
+This section describes the basic data model of how results of discovery services are
+stored and how new discovery services can extend and enrich this model.
+
+See the section [ODF Metadata API](odf-metadata-api.html) for general information
+on how to retrieve metadata.
+
+You can find the current Atlas data model in the file
+
+ odf-core/src/main/resources/org/apache/atlas/odf/core/metadata/internal/atlas/atlas-odf-model.json
+
+which contains JSON that can be POSTed to the Atlas `types` REST resource to create those types.
+
+## Annotations
+
+All discovery services results are called "annotations". An annotation is an object
+that annotates another object with a certain piece of information.
+For instance, you could have a `DataClassAnnotation` which has a reference attribute `classifiedObject` linking it to
+a column `NAME` of a table `CUSTREC`, a list of reference attributes `classifyingObjects` linking it to business terms.
+An additional attribute `confidences` might provide a list of numeric confidence values indicating the "strength" of the
+relationship between the classifiedObject and the respective classifyingObject (business term).
+ For column `NAME` the list of classifying objects may have a single entry `Customer Name` the list of confidence values
+ may have a single value `0.7`.
+This annotation expresses the fact that the term classification services registered and active in ODF have come up with a
+70% confidence of CUSTREC.NAME representing a customer name.
+
+Technically, an annotation is a subtype of one of the three *base types* which are subtyes of the (abstract)
+Atlas type `Annotation`:
+
+- `ProfilingAnnotation`
+- `ClassificationAnnotation`
+- `RelationshipAnnotation`
+
+
+A `ProfilingAnnotation` assigns non-reference attributes to an object. It has the following non-reference attributes:
+
+- `annotationType`: The type of annotation. A Json string of the form
+ `{"stdType": "DataQualityAnnotation",`
+ ` "runtime": "JRE",`
+ ` "spec" : “org.apache.atlas.myservice.MyAnnotation"`
+ `}`
+ where `stdType` is a base or standardized type name (see below for standardized types), `runtime` names the runtime,
+ and `spec` is a runtime-specific string which helps the runtime to deal with instances of this type. In case of a Java
+ runtime the `spec` is the name of the implementing class which may be a subclass of the `stdType`.
+- `analysisRun`: A string that is set to the request Id of the analysis that created it.
+(Compare the swagger documentation of the REST resource `analyses`, e.g. [here](https://sdp1.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT/swagger/#/analyses).
+*Internal Note*: will be replaced with RID
+- `summary`: A human-readable string that presents a short summary of the annotation.
+Might be used in generic UIs for displaying unknown annotations.
+*Internal note*: deprecated
+- `jsonProperties`: A string attributes where you can store arbitrary JSON as a string.
+Can be used to 'extend' standard annotations.
+
+...and a single referencing attribute:
+
+- `profiledObject`: The object that is annotated by this annotation. In the example above,
+this would point to the Column object.
+
+
+A `ClassificationAnnotation` assigns any number (including 0) of meta data objects to an object.
+It has the same non-reference attributes as `ProfilingAnnotation` plus the following reference attributes:
+
+- `classifiedObject`: The object that is annotated by this annotation.
+- `classifyingObjects`: List of references to meta data objects classifying the classifiedObject.
+
+A `RelationshipAnnotation` expresses a relationship between meta data objects.
+It has the same non-reference attributes as `ProfilingAnnotation` plus a single reference attribute:
+
+- `relatedObjects`: List of references to related meta data objects.
+
+
+Note that annotations are implemented as proper Atlas object types and not traits (labels) for these reasons:
+
+- Annotations of the same type but of different discovery service should be able co-exist, for instance,
+to be able to compare results of different services downstream.
+This is only partly possible with traits.
+- Relationships between objects can not easily be modeled with traits.
+
+A discovery service can deliver its results in a base, standardized, or *custom annotation type*. Depending on the type of the
+underlying relationship a custom annotation type might be a subtype of `ProfilingAnnotation` (asymmetric, single reference attribute),
+`ClassificationAnnotation` (asymmetric, any number of reference attributes), or `RelationshipAnnotation` (symmetric, any number
+of reference attributes). A custom annotation type can have additional non-reference attributes that are stored in its `jsonProperties`.
+
+When implemented in Java, the class defining a custom annotation has private fields and corresponding getter/setter methods
+representing the additional information.
+
+
+##Example
+
+
+For instance, creating a new annotation of type `org.apache.atlas.oli.MyAnnotation` could look like this.
+
+ public class MyAnnotation extends ClassificationAnnotation {
+ String myNewAttribute;
+
+ public String getMyNewAttribute() {
+ return myNewAttribute;
+ }
+
+ public void setMyNewAttribute(String myNewAttribute) {
+ this.myNewAttribute = myNewAttribute;
+ }
+ }
+
+Annotations can be mapped into standardized meta data objects by a *propagator* which implements the `AnnotationPropagator` interface.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/discovery-service-tutorial.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/discovery-service-tutorial.md b/odf/odf-doc/src/site/markdown/discovery-service-tutorial.md
new file mode 100755
index 0000000..2ab8775
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/discovery-service-tutorial.md
@@ -0,0 +1,157 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Tutorial: Build and run your first Discovery Service
+This tutorial shows how you can create your first discovery service in Java that analyzes a data set and creates a single annotation of a new type.
+This tutorial requires that you have [Maven](http://maven.apache.org/) installed.
+
+
+## Create a discovery service
+Follow these steps to create and package a Java implementation of the simplest discovery service.
+
+#### Step 1: Create ODF discovery service maven project
+Create a new Java Maven project from the ODF provided archetype ``odf-archetype-discoveryservice`` (group ID ``org.apache.atlas.odf``).
+Choose the following values for the respective parameters:
+
+| Parameter | Value |
+|-----------|--------------------------|
+|groupId | odftutorials |
+|artifactId | discoveryservicetutorial |
+|version | 0.1 |
+
+
+From the command line, your command may look like this:
+
+ mvn archetype:generate -DarchetypeGroupId=org.apache.atlas.odf -DarchetypeArtifactId=odf-archetype-discoveryservice -DarchetypeVersion=0.1.0-SNAPSHOT -DgroupId=odftutorials -DartifactId=discoveryservicetutorial -Dversion=0.1
+
+This will create a new Maven project with a pom that has dependencies on ODF.
+It will also create two Java classes ``MyDiscoveryService`` and ``MyAnnotation``
+that you may want to use as a basis for the following steps.
+
+If you use Eclipse to create your project, be sure to enable the checkbox "Include snapshot archetypes" in the
+archetype selection page of the New Maven Project wizard.
+
+If you are not interested in the actual code at this point, you may skip Steps 2 through 4 and go directly
+to step 5.
+
+#### Step 2 (optional): Check the discovery service implementation class
+Create a new Java class named ``odftutorials.MyDiscoveryService`` that inherits from `org.apache.atlas.odf.core.discoveryservice.SyncDiscoveryServiceBase`.
+As the interface name indicates, our service will be synchronous, i.e., it will have a simple method ``runAnalysis()`` that returns
+the analysis result. For the implementation of long-running, asynchronous services, see TODO.
+The archetype creation has already filled in some code here that we will use. Your class
+should look something like this:
+
+ public class MyDiscoveryService extends SyncDiscoveryServiceBase {
+
+ @Override
+ public DiscoveryServiceSyncResponse runAnalysis(DiscoveryServiceRequest request) {
+ // 1. create an annotation that annotates the data set object passed in the request
+ MyAnnotation annotation = new MyAnnotation();
+ annotation.setAnnotatedObject(request.getDataSetContainer().getDataSet().getReference());
+ // set a new property called "tutorialProperty" to some string
+ annotation.setMyProperty("My property was created on " + new Date());
+
+ // 2. create a response with our annotation created above
+ return createSyncResponse( //
+ ResponseCode.OK, // Everything works OK
+ "Everything worked", // human-readable message
+ Collections.singletonList(annotation) // new annotations
+ );
+ }
+ }
+
+What does the code do?
+The code basically consists of two parts:
+
+1. Create a new ``MyAnnotation`` object and annotate the data set that is passed into
+the discovery service with it.
+2. Create the discovery service response with the new annotation and return it.
+
+
+#### Step 3 (optional): Check the new annotation class
+The project also contains a new Java class called ``odftutorials.MyAnnotation``
+which extends the class ``org.apache.atlas.odf.core.metadata.ProfilingAnnotation``.
+It is a new annotation type that contains a property called ``myProperty`` of type ``String``.
+In the code you can see that there is a Java-Bean style getter and a setter method, i.e., ``getTutorialProperty()`` and
+``setTutorialProperty(String value)``.
+
+ public class MyAnnotation extends ProfilingAnnotation {
+
+ private String myProperty;
+
+ public String getMyProperty() {
+ return myProperty;
+ }
+
+ public void setMyProperty(String myValue) {
+ this.myProperty = myValue;
+ }
+ }
+
+When we return these annotations, ODF will take care that these annotations are stored
+appropriately in the metadata store.
+
+
+#### Step 4 (optional): Check the discovery service descriptor
+Lastly, the project contains a file called ``META-INF/odf/odf-services.json``
+in the ``main/resources`` folder. This file which always have to have the same
+name contains a JSON list of the the descriptions of all services defined in our project.
+The descriptions contain an ID, a name, a short human-readable description, together
+with the Java class name implementing the service. Here is how it looks like:
+
+ [
+ {
+ "id": "odftutorials.discoveryservicetutorial.MyDiscoveryService",
+ "name": "My service",
+ "description": "My service creates my annotation for a data set",
+ "type": "Java",
+ "endpoint": "odftutorials.MyDiscoveryService"
+ }
+ ]
+
+Note that most of this information can be changed but ``type`` (this is a Java implementation)
+and ``endpoint`` (the Java class is called ``odftutorials.MyDiscoveryService``)
+should remain as the are.
+
+#### Step 5: Build the service JAR
+The service jar is a standard jar file so you can build it with a standard Maven command like
+
+ mvn clean install
+
+You can find the output jar as per the Maven convention in
+``target/discoveryservicetutorial-0.1.jar``
+
+
+## Deploy the discovery service
+Once you've built your service JAR as described in the previous section, there are two ways
+how you can deploy it.
+
+### Classpath deployment
+The simplest way to make an ODF instance pickup your new service is add the service JAR (and
+any dependent JARs) to the ODF classpath. A simple way to do this is to package the JARs into
+ODF war file. Once you (re-)start ODF, your new service should be available.
+
+## Run the discovery service
+
+Perform these steps to run your new service and inspect the results.
+
+1. Go to the Analysis tab in the ODF console
+2. Select the Data Sets tab and click on Start Analysis next to any data set
+3. Select "My Service" as the discovery service and click Submit.
+4. Select the Requests tab and click Refresh
+5. You should see a new entry showing the data set and the "My Service" discovery service.
+6. Click on Annotations. A new page will open that opens the Atlas UI with a list of the new
+annotation that was created.
+7. Click on the annotation and check the value of the "myProperty" property. It should contain
+a value like ``My property was created on Mon Feb 01 18:31:51 CET 2016``.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/discovery-services.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/discovery-services.md b/odf/odf-doc/src/site/markdown/discovery-services.md
new file mode 100755
index 0000000..1afa36c
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/discovery-services.md
@@ -0,0 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Discovery services
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/examples.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/examples.md b/odf/odf-doc/src/site/markdown/examples.md
new file mode 100755
index 0000000..084af75
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/examples.md
@@ -0,0 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Examples
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/first-analysis-tutorial.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/first-analysis-tutorial.md b/odf/odf-doc/src/site/markdown/first-analysis-tutorial.md
new file mode 100755
index 0000000..b299bfa
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/first-analysis-tutorial.md
@@ -0,0 +1,17 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Run your first ODF analysis
+
+See the [First steps](first-steps.html) section for details on how to run the analysis from the ODF console UI.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/first-steps.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/first-steps.md b/odf/odf-doc/src/site/markdown/first-steps.md
new file mode 100755
index 0000000..00ced52
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/first-steps.md
@@ -0,0 +1,77 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# First Steps
+
+This section assumes that you have ODF installed, either the
+[test environment](test-env.html) or [manually](install.html).
+
+
+### ODF Console UI
+
+To open the ODF console point your browser to the ODF web application.
+In the [test environment](test-env.html), this is typically
+[https://localhost:58081/odf-web-0.1.0-SNAPSHOT](https://localhost:58081/odf-web-0.1.0-SNAPSHOT).
+
+*Note*: The links to the ODF Console in the instructions below only work if you view this documentation
+from the ODF web application.
+
+The default user of the ODF Console is odf / admin4odf.
+
+
+#### Check System health
+Go to the [System monitor](/odf-web-0.1.0-SNAPSHOT/#monitor) tab and
+click "Check health". After a while you should see a message
+that the health check was successful. If this check fails it might be the case that the dependent services
+are not fully up and running yet (typically happens after start of the test environment), so wait a short while
+and try again.
+
+#### Discovery Services
+Take a look at all available discovery services on the tab [Discovery Services](/odf-web-0.1.0-SNAPSHOT/#discoveryServices).
+
+#### Configure Atlas repository and create sample metadata
+
+To change the URL of your Atlas installation and to create some sample data, go
+to the [Configuration](/odf-web-0.1.0-SNAPSHOT/#configuration) tab.
+In general, it is a good idea change the default URL to Atlas from "localhost" to a hostname that is accessible
+from your network. If you don't do this, you might experience some strange effects when viewing
+Atlas annotations from the web app.
+If you changed the name, click Save.
+
+Create a set of simple sample data by clicking on Create Atlas Sample Data.
+
+To explore the sample data go to the [Data Sets](/odf-web-0.1.0-SNAPSHOT/#data) tab.
+
+#### Run analysis
+
+The easiest way to start an analysis is from the [Data Sets](/odf-web-0.1.0-SNAPSHOT/#data) tab.
+In the "Data Files" section look for the sample table "BankClientsShort".
+To view the details of the table click on it anywhere in the row. The Details dialog
+shows you information about this data set. Click Close to close the dialog.
+
+To start an analysis on the "BankClientsShort" table click "Start Analysis" on the right.
+In the "New Analysis Request" dialog click on "<Select a Service>" to add a service to
+the sequence of discovery service to run on the data set. Then click "Submit" to start the analysis.
+
+To check the status of your request go to the
+[Analysis](/odf-web-0.1.0-SNAPSHOT/#analysis) tab and click Refresh.
+If all went well the status is "Finished".
+Click on "View Results" to view all annotations created for this analysis request.
+
+
+### REST API
+See the [REST API documentation](/odf-web-0.1.0-SNAPSHOT/swagger) for more details on how to
+perform the actions explained above with the REST API.
+In particular, have a look at the ``analysis`` REST resource for APIs how to start and
+monitor analyis requests.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/index.md b/odf/odf-doc/src/site/markdown/index.md
new file mode 100755
index 0000000..4c7656c
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/index.md
@@ -0,0 +1,23 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Overview
+
+The "Open Discovery Framework" is an open metadata-based framework that strives to be a common home for different analytics technologies that discover characteristics of data sets and relationships between them (think "AppStore for discovery algorithms"). Using ODF, applications can leverage new discovery algorithms and their results with minimal integration effort.
+
+Automated characterization of information and automated discovery of relationships is key to several Analytics Platform initiatives, e.g. enable and improve self service for knowledge workers.
+The Open Discovery Platform provides infrastructure based on open source technology to easily execute, manage and integrate diverse metadata discovery algorithms provided by internal
+or external (open source) contributors in a single point of access. These discovery algorithms store their analysis results in a common open metadata
+repository that promotes reuse and sharing of these results.
+A simple plug-in mechanism to integrate discovery services enables users of ODF to easily combine and orchestrate algorithms built on different technologies,
+thereby gaining deeper insights into their data.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/install.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/install.md b/odf/odf-doc/src/site/markdown/install.md
new file mode 100755
index 0000000..8c521ba
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/install.md
@@ -0,0 +1,149 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Install ODF manually
+
+This section describes how to manually install ODF and its prerequisites.
+
+## Install ODF locally
+
+ODF is installed on an application server like jetty. Its prereqs (Kafka and Atlas)
+can run on separate machines, they simple must be reachable over the network.
+
+ODF's configuration is stored in Zookeeper which is a prereq for Kafka.
+
+
+### Prerequisites
+
+ODF has two prerequisites:
+
+1. Apache Atlas (only tested with 0.6 but no hard dependency required here)
+2. Apache Kafka 0.8.2.1 which, in turn, requires Zookeper 3.4
+
+#### Apache Atlas
+
+[Apache Atlas](http://atlas.incubator.apache.org/) is an open
+metadata infrastructure that is currently in incubator status.
+There is currently no binary download available, you have to [build it yourself](http://atlas.incubator.apache.org/InstallationSteps.html).
+Alternatively, you can download a version that we built [here](https://ibm.box.com/shared/static/of1tdea7465iaen8ywt7l1h761j0fplt.zip).
+
+After you built the distribution, simply unpack the tar ball, and run
+``bin/atlas_start.py``. Atlas will be started on port 21443 as default, so point
+your browser to [https://localhost:21443](https://localhost:21443) to look at the
+Atlas UI.
+Note that starting Atlas can take up to a minute.
+
+To stop, run ``bin/atlas_stop.py``.
+
+See the Atlas section in the [Troubleshooting guide](troubleshooting.html)
+for common issues and how to workaround them.
+
+#### Apache Kafka
+
+[Apache Kafka](http://kafka.apache.org/) is an open source project that implements
+a messaging infrastructure. ODF uses Kafka for notifications and queueing up requests to
+discovery services.
+
+To install Kafka, download the version 0.8.2.1 with Scala 2.10 (which is the version we used in
+our tests) from the Kafka website, see [here](https://www.apache.org/dyn/closer.cgi?path=/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz).
+
+After unpacking the tar ball these steps should get you going:
+
+1. CD to the distribution directory.
+2. Start zookeeper first by running ``bin/zookeeper-server-start.sh config/zookeeper.properties``.
+3. Start Kafka: ``bin/kafka-server-start.sh config/server.properties``
+
+By default, Zookeeper is running on port 2181 and Kafka runs on port 9092. You can change the Zookeeper
+port by changing the properties ``clientPort`` in ``config/zookeeper.properties`` and
+``zookeeper.connect`` in ``config/server.properties``. Change the Kafka port by changing
+``port`` in ``config/server.properties``.
+
+For Windows, run the respective .bat commands in the ``bin\windows`` directory.
+
+
+### Deploy ODF
+
+The only ODF artifact you need for deployment is the war file built by the odf-web maven project, which can typically
+be found here:
+
+ odf-web/target/odf-web-0.1.0-SNAPSHOT.war
+
+To tell ODF which Zookeeper / Kafka to use you will need to set the
+Java system property ``odf.zookeeper.connect`` to point
+to the Zookeeper host and port. The value is typically the same string as the ``zookeeper.connect`` property
+in the Kafka installation ``config/server.properties`` file:
+
+ -Dodf.zookeeper.connect=zkserver.example.org:2181
+
+Note that if this property is not set, the default is ``localhost:52181``.
+
+
+#### Application Server
+
+ODF should run on any application server. As of now we have done most of our testing on Jetty.
+
+##### Jetty
+
+[Jetty](https://eclipse.org/jetty/) is an open source web and application server.
+We have used version 9.2.x for our testing (the most current one that supports Java 7).
+Download it from the web site [https://eclipse.org/jetty/](https://eclipse.org/jetty/).
+
+Here are some quick start instructions for creating a new Jetty base. Compare
+the respective Jetty documentation section [here](http://www.eclipse.org/jetty/documentation/9.2.10.v20150310/quickstart-running-jetty.html#creating-jetty-base).
+
+First, for in order to enable basic authentication, the following configuration needs to be added to the `etc/jetty.xml` file, right before the closing `</Configure>` tag at the end of the file:
+
+```
+<Call name="addBean">
+ <Arg>
+ <New class="org.eclipse.jetty.security.HashLoginService">
+ <Set name="name">ODF Realm</Set>
+ <Set name="config"><SystemProperty name="jetty.home" default="."/>/etc/realm.properties</Set>
+ </New>
+ </Arg>
+</Call>
+```
+
+Secondly, a `etc/realm.properties` file needs to be added that contains the credentials of the ODF users in the following [format](http://www.eclipse.org/jetty/documentation/9.2.10.v20150310/configuring-security-authentication.html#security-realms):
+
+```
+<username>: <password>[,<rolename> ...]
+```
+
+Then, you will have to create and initialize new directory where you deploy your web apps and
+copy the ODF war there. These commands should do the trick:
+
+ mkdir myjettybase
+ cd myjettybase
+ java -jar $JETTY_HOME\start.jar --add-to-startd=https,ssl,deploy
+ cp $ODFDIR/odf-web-0.1.0-SNAPSHOT.jar webapps
+ java -Dodf.zookeeper.connect=zkserver.example.org:2181 -jar $JETTY_HOME\start.jar
+
+The first java command initializes the jetty base directory by creating a directory ``start.d`` which
+contains some config files (e.g. http.ini contains the port the server runs on) and the
+empty ``webapps`` directory.
+The copy command copies the ODF war file to the webapps folder.
+The last command starts Jetty (on default port 8443). You can stop it by hitting Ctrl-C.
+
+You should see a message like this one indicating that the app was found and started.
+
+ 2016-02-26 08:28:24.033:INFO:oejsh.ContextHandler:Scanner-0: Started o.e.j.w.WebAppContext@-545d793e{/odf-web-0.1.0-SNAPSHOT,file:/C:/temp/jetty-0.0.0.0-8443-odf-web-0.1.0-SNAPSHOT.war-_odf-web-0.1.0-SNAPSHOT-any-8485458047819836926.dir/webapp/,AVAILABLE}{myjettybase\webapps\odf-web-0.1.0-SNAPSHOT.war}
+
+Point your browser to [https://localhost:8443/odf-web-0.1.0-SNAPSHOT](https://localhost:8443/odf-web-0.1.0-SNAPSHOT) to see the ODF console.
+
+
+
+##### Websphere Liberty Profile
+
+Stay tuned
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/jenkins-build.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/jenkins-build.md b/odf/odf-doc/src/site/markdown/jenkins-build.md
new file mode 100755
index 0000000..1b22bb2
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/jenkins-build.md
@@ -0,0 +1,93 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# ODF Jenkins build
+
+## General
+The Jenkins build is set up at
+[https://shared-discovery-platform-jenkins.swg-devops.com:8443](https://shared-discovery-platform-jenkins.swg-devops.com:8443).
+
+### Available jobs
+The following jobs are available:
+
+1. **Open-Discovery-Framework**: The main build on the master branch with the test environment.
+Built on Linux.
+2. **Open-Discovery-Framework-Parameters**: Job you can trigger manually for private branches
+and platforms. Using the `nodelabel` parameter with value `odfbuild` triggers the build on Linux.
+3. **Open-Discovery-Framework-Testenv**: Manages and/or installs the test env. This job is currently scheduled
+to install the current testenv on the [machine associated with label `odftestenv`](http://sdp1.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT) every night at 10PM EST.
+
+The parameter `nodelabel` defines the nodes the test env is / should be installed:
+
+- `odftestenv` for testing your private builds on [https://sdp1.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT](https://sdp1.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT).
+- `odfdemo` for the stable demo on [https://odfdemo.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT](https://odfdemo.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT).
+
+Possible actions selectable through the `action` parameter are:
+
+- `start`: (re)start the test env
+- `stop`: stop the test env
+- `cleanconfig`: (re)starts with clean configuration and Kafka topics
+- `cleanmetadata`: (re)starts with clean metadata
+- `cleanall`: (re)starts with cleanconfig plus cleanmetadata
+- `install`: Installs the build as specified in the `jenkinsjob` and `buildnumber` parameters.
+
+4. **Open-Discovery-Framework-BuildStarter**: Job polling for changes in the master branch and triggering
+the automated build. Starts the Open-Discovery-Framework Linux build. *You should typically not have to trigger this job manually!*
+
+You can find these jobs in Jenkins in the [1-ODF](https://shared-discovery-platform-jenkins.swg-devops.com:8443/view/1-ODF/) tab.
+
+### Node labels
+This Jenkins system currently contains two kinds of slaves which are distinguished by a
+so called [node label](https://www.safaribooksonline.com/library/view/jenkins-the-definitive/9781449311155/ch11s04.html).
+
+We currently have these node labels:
+
+1. `odfbuild`: Linux build
+2. `odftestenv`: Machine sdp1.rtp.raleigh.ibm.com where test envs can be deployed regularly for internal testing.
+
+
+### Some Important Settings
+
+- Use the profile `jenkinsbuild`. This is currently only used in the Bluemix Services and requires that the Bluemix password is not read from the `cf.password` system property but rather from the env var `CFPASSWORD`. This is only done so that the password doesn't appear in the log.
+- The firewall is smashed with a script called `smashlittletonfirewall.sh` (see below). You have to set the env var
+`INTRANETCREDENTIALS` from Jenkins as a combined credential variable (of the form user:password). The reason
+why this is a script and not put into the command line directly is that the user / password don't appear in the log
+
+
+### Build Slave Machines
+The build slave machines are:
+
+1. BuildNode: `sdp1.rtp.raleigh.ibm.com`
+2. BuildNode2: `sdpbuild2.rtp.raleigh.ibm.com`
+3. ODFTestEnv: `sdpdemo.rtp.raleigh.ibm.com`
+4. BuildNodeWin1: `sdpwin1.rtp.raleigh.ibm.com`
+
+Access user: ibmadmin / adm4sdp
+
+These VMs can be managed through [vLaunch](https://vlaunch.rtp.raleigh.ibm.com/).
+
+
+### Scripts / settings required on the build slave
+
+#### Windows
+On the windows slaves, install Git from IBM iRAM, e.g., [here](https://w3-03.ibm.com/tools/cm/iram/oslc/assets/503004E8-5971-230E-3D16-6F3FBDBE2E2C/2.5.1)
+and make sure that the *bin* directory of the installation (typically something like `C:\Program Files (x86)\Git\bin`) is in the path.
+This takes care that `sh.exe` is in the path and picked up by the Jenkins jobs.
+
+#### `smashlittletonfirewall.sh`
+
+Used to smash the Littleton firewall. Put this somewhere in the path, e.g., `~/bin`. The reason why this exists
+at all is so that the intranet credentials don't appear in the build log. The file consists of this one line:
+
+ curl -i -L --user $INTRANETCREDENTIALS --insecure -X GET http://ips-rtc.swg.usma.ibm.com/jazz/web/projects
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/odf-metadata-api.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/odf-metadata-api.md b/odf/odf-doc/src/site/markdown/odf-metadata-api.md
new file mode 100755
index 0000000..6c5fa91
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/odf-metadata-api.md
@@ -0,0 +1,63 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# ODF Metadata API
+
+ODF provides a very simple API for searching, retrieving, and (to a limited extent) for creating
+new metadata objects.
+This API abstracts away specifics of the underlying metadata store.
+See the REST resource `metadata`, e.g., [here](https://sdp1.rtp.raleigh.ibm.com:58081/odf-web-0.1.0-SNAPSHOT/swagger/#/metadata) or look at the Java interface `org.apache.atlas.odf.core.metadata.MetadataStore`.
+
+In this API we distinguish between `MetaDataObject`s and `MetadataObjectReferences`.
+Where the former represent an object as such, the latter is just a reference to an object.
+You may think of the `MetaDataObjectReference` as a generalized XMeta RID.
+
+Simply put, metadata objects are represented as JSON where object attributes
+are represented as JSON attribute with the same name.
+Simple types map to JSON simple types. References
+to another object are represented of JSON objects of type `MetadataObjectReference` that
+has three attribute:
+1. `id`: the object ID
+2. `repositoryId`: the ID of the repository where the object resides
+3. `url` (optional): A URL pointing to the object. For Atlas, this is a link to the object in the
+Atlas dashboard.
+
+The API is read-only, the only objects that can be created are annotations (see section [Data model and extensibility](data-model.html).
+
+Here is an example: suppose there is a table object which has a name and a list of columns. The JSON of this table would look something like this:
+
+ {
+ "name": "CUSTOMERS,
+ "columns": [
+ {
+ "id": "1234-abcd",
+ "repositoryId": "atlas:repos1"
+ },
+ {
+ "id": "5678-efgh",
+ "repositoryId": "atlas:repos1"
+ }
+ ],
+ "reference": {
+ "id": "9abc-ijkl",
+ "repositoryId": "atlas:repos1"
+ },
+ "javaClass": "corg.apache.atlas.odf.core.metadata.models.Table"
+ }
+
+The `reference` value represent the reference to the object itself where as
+`javaClass` denotes the type of object (table in this case).
+The `name` attribute contains the table name where the `columns` value is a list
+of references to two column objects. These references can be retrieved separately
+to look at the details.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/6d19e129/odf/odf-doc/src/site/markdown/operations.md
----------------------------------------------------------------------
diff --git a/odf/odf-doc/src/site/markdown/operations.md b/odf/odf-doc/src/site/markdown/operations.md
new file mode 100755
index 0000000..f28699b
--- /dev/null
+++ b/odf/odf-doc/src/site/markdown/operations.md
@@ -0,0 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Operations