You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sentry.apache.org by va...@apache.org on 2014/04/29 23:24:15 UTC
[2/3] git commit: SENTRY-186: e2e tests for solr document-level
security (Gregory Chanan via Vamsee Yarlagadda)
SENTRY-186: e2e tests for solr document-level security (Gregory Chanan via Vamsee Yarlagadda)
Project: http://git-wip-us.apache.org/repos/asf/incubator-sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-sentry/commit/b4c39a1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-sentry/tree/b4c39a1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-sentry/diff/b4c39a1e
Branch: refs/heads/master
Commit: b4c39a1e89bd1f640d01e31d83d4833602e17a25
Parents: 574a685
Author: Vamsee <va...@cloudera.com>
Authored: Tue Apr 29 13:56:44 2014 -0700
Committer: Vamsee <va...@cloudera.com>
Committed: Tue Apr 29 13:56:44 2014 -0700
----------------------------------------------------------------------
.../e2e/solr/AbstractSolrSentryTestBase.java | 20 +-
.../tests/e2e/solr/TestDocLevelOperations.java | 289 +++
.../resources/solr/collection1/conf/schema.xml | 2 +
.../collection1/conf/solrconfig-doclevel.xml | 1881 ++++++++++++++++++
.../solr/sentry/test-authz-provider.ini | 7 +-
5 files changed, 2193 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/b4c39a1e/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
index d58f3b9..e90891d 100644
--- a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
+++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/AbstractSolrSentryTestBase.java
@@ -75,6 +75,7 @@ public class AbstractSolrSentryTestBase {
protected static final String ALL_DOCS = "*:*";
protected static final Random RANDOM = new Random();
protected static final String RESOURCES_DIR = "target" + File.separator + "test-classes" + File.separator + "solr";
+ protected static final String CONF_DIR_IN_ZK = "conf1";
private static final int NUM_SERVERS = 4;
private static void addPropertyToSentry(StringBuilder builder, String name, String value) {
@@ -164,7 +165,7 @@ public class AbstractSolrSentryTestBase {
* @return - the username as String
* @throws Exception
*/
- private String getAuthenticatedUser() throws Exception {
+ protected String getAuthenticatedUser() throws Exception {
return ModifiableUserAuthenticationFilter.getUser();
}
@@ -675,12 +676,23 @@ public class AbstractSolrSentryTestBase {
verifyUpdatePass(ADMIN_USER, collectionName, solrInputDoc);
}
- protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception {
+ private ZkController getZkController() {
SolrDispatchFilter dispatchFilter =
(SolrDispatchFilter) miniSolrCloudCluster.getJettySolrRunners().get(0).getDispatchFilter().getFilter();
- ZkController zkController = dispatchFilter.getCores().getZkController();
+ return dispatchFilter.getCores().getZkController();
+ }
+
+ protected void uploadConfigDirToZk(String collectionConfigDir) throws Exception {
+ ZkController zkController = getZkController();
// conf1 is the config used by AbstractFullDistribZkTestBase
- zkController.uploadConfigDir(new File(collectionConfigDir), "conf1");
+ zkController.uploadConfigDir(new File(collectionConfigDir),
+ CONF_DIR_IN_ZK);
+ }
+
+ protected void uploadConfigFileToZk(String file, String nameInZk) throws Exception {
+ ZkController zkController = getZkController();
+ zkController.getZkClient().makePath(ZkController.CONFIGS_ZKNODE + "/"
+ + CONF_DIR_IN_ZK + "/" + nameInZk, new File(file), false, true);
}
protected CloudSolrServer createNewCloudSolrServer() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/b4c39a1e/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java
new file mode 100644
index 0000000..eccc576
--- /dev/null
+++ b/sentry-tests/sentry-tests-solr/src/test/java/org/apache/sentry/tests/e2e/solr/TestDocLevelOperations.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.tests.e2e.solr;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.junit.After;
+import org.junit.Before;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.io.IOUtils;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.util.EntityUtils;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.ByteArrayOutputStream;
+import java.net.URI;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.junit.Test;
+
+/**
+ * Test the document-level security features
+ */
+public class TestDocLevelOperations extends AbstractSolrSentryTestBase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestDocLevelOperations.class);
+ private static final String DEFAULT_COLLECTION = "collection1";
+ private static final String AUTH_FIELD = "sentry_auth";
+ private static final int NUM_DOCS = 100;
+ private static final int EXTRA_AUTH_FIELDS = 2;
+ private String userName = null;
+
+ @Before
+ public void beforeTest() throws Exception {
+ userName = getAuthenticatedUser();
+ }
+
+ @After
+ public void afterTest() throws Exception {
+ setAuthenticationUser(userName);
+ }
+
+ private void setupCollectionWithDocSecurity(String name) throws Exception {
+ String configDir = RESOURCES_DIR + File.separator + DEFAULT_COLLECTION
+ + File.separator + "conf";
+ uploadConfigDirToZk(configDir);
+ // replace solrconfig.xml with solrconfig-doc-level.xml
+ uploadConfigFileToZk(configDir + File.separator + "solrconfig-doclevel.xml",
+ "solrconfig.xml");
+ setupCollection(name);
+ }
+
+ private String doHttpGet(CloudSolrServer server, String path) throws Exception {
+ HttpClient httpClient = server.getLbServer().getHttpClient();
+ Set<String> liveNodes =
+ server.getZkStateReader().getClusterState().getLiveNodes();
+ assertTrue("Expected at least one live node", !liveNodes.isEmpty());
+ String firstServer = liveNodes.toArray(new String[0])[0].replace("_solr", "/solr");
+ URI uri = new URI("http://" + firstServer + path);
+ HttpGet get = new HttpGet(uri);
+ HttpEntity httpEntity = null;
+ boolean success = false;
+ String retValue = "";
+ try {
+ final HttpResponse response = httpClient.execute(get);
+ int httpStatus = response.getStatusLine().getStatusCode();
+ httpEntity = response.getEntity();
+
+ if (httpEntity != null) {
+ InputStream is = httpEntity.getContent();
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ try {
+ IOUtils.copyLarge(is, os);
+ os.flush();
+ } finally {
+ IOUtils.closeQuietly(os);
+ IOUtils.closeQuietly(is);
+ }
+ retValue = os.toString();
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ EntityUtils.consumeQuietly(httpEntity);
+ get.abort();
+ }
+ }
+ return retValue;
+ }
+
+ /**
+ * Test that queries from different users only return the documents they have access to.
+ */
+ @Test
+ public void testDocLevelOperations() throws Exception {
+ String collectionName = "docLevelCollection";
+ setupCollectionWithDocSecurity(collectionName);
+
+ // create documents
+ ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
+ for (int i = 0; i < NUM_DOCS; ++i) {
+ SolrInputDocument doc = new SolrInputDocument();
+ String iStr = Long.toString(i);
+ doc.addField("id", iStr);
+ doc.addField("description", "description" + iStr);
+
+ // put some bogus tokens in
+ for (int k = 0; k < EXTRA_AUTH_FIELDS; ++k) {
+ doc.addField(AUTH_FIELD, AUTH_FIELD + Long.toString(k));
+ }
+ // 50% of docs get "junit", 50% get "admin" as token
+ if (i % 2 == 0) {
+ doc.addField(AUTH_FIELD, "junit");
+ } else {
+ doc.addField(AUTH_FIELD, "admin");
+ }
+ // add a token to all docs so we can check that we can get all
+ // documents returned
+ doc.addField(AUTH_FIELD, "docLevel");
+
+ docs.add(doc);
+ }
+ CloudSolrServer server = getCloudSolrServer(collectionName);
+ try {
+ server.add(docs);
+ server.commit(true, true);
+
+ // queries
+ SolrQuery query = new SolrQuery();
+ query.setQuery("*:*");
+
+ // as junit -- should get half the documents
+ setAuthenticationUser("junit");
+ QueryResponse rsp = server.query(query);
+ SolrDocumentList docList = rsp.getResults();
+ assertEquals(NUM_DOCS / 2, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ assertEquals(0, Long.valueOf(id) % 2);
+ }
+
+ // as admin -- should get the other half
+ setAuthenticationUser("admin");
+ rsp = server.query(query);
+ docList = rsp.getResults();
+ assertEquals(NUM_DOCS / 2, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ assertEquals(1, Long.valueOf(id) % 2);
+ }
+
+ // as docLevel -- should get all
+ setAuthenticationUser("docLevel");
+ rsp = server.query(query);
+ assertEquals(NUM_DOCS, rsp.getResults().getNumFound());
+
+ // test filter queries work as AND -- i.e. user can't avoid doc-level
+ // checks by prefixing their own filterQuery
+ setAuthenticationUser("junit");
+ String fq = URLEncoder.encode(" {!raw f=" + AUTH_FIELD + " v=docLevel}");
+ String path = "/" + collectionName + "/select?q=*:*&fq="+fq;
+ String retValue = doHttpGet(server, path);
+ assertTrue(retValue.contains("numFound=\"" + NUM_DOCS / 2 + "\" "));
+
+ // test that user can't inject an "OR" into the query
+ final String syntaxErrorMsg = "org.apache.solr.search.SyntaxError: Cannot parse";
+ fq = URLEncoder.encode(" {!raw f=" + AUTH_FIELD + " v=docLevel} OR ");
+ path = "/" + collectionName + "/select?q=*:*&fq="+fq;
+ retValue = doHttpGet(server, path);
+ assertTrue(retValue.contains(syntaxErrorMsg));
+
+ // same test, prefix OR this time
+ fq = URLEncoder.encode(" OR {!raw f=" + AUTH_FIELD + " v=docLevel}");
+ path = "/" + collectionName + "/select?q=*:*&fq="+fq;
+ retValue = doHttpGet(server, path);
+ assertTrue(retValue.contains(syntaxErrorMsg));
+ } finally {
+ server.shutdown();
+ }
+ }
+
+ /**
+ * Test the allGroupsToken. Make it a keyword in the query language ("OR")
+ * to make sure it is treated literally rather than interpreted.
+ */
+ @Test
+ public void testAllGroupsToken() throws Exception {
+ String allGroupsToken = "OR";
+ String collectionName = "allGroupsCollection";
+ setupCollectionWithDocSecurity(collectionName);
+
+ int junitFactor = 2;
+ int allGroupsFactor = 5;
+
+ int totalJunitAdded = 0; // total docs added with junit token
+ int totalAllGroupsAdded = 0; // total number of docs with the allGroupsToken
+ int totalOnlyAllGroupsAdded = 0; // total number of docs with _only_ the allGroupsToken
+
+ // create documents
+ ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
+ for (int i = 0; i < NUM_DOCS; ++i) {
+ boolean addedViaJunit = false;
+ SolrInputDocument doc = new SolrInputDocument();
+ String iStr = Long.toString(i);
+ doc.addField("id", iStr);
+ doc.addField("description", "description" + iStr);
+
+ if (i % junitFactor == 0) {
+ doc.addField(AUTH_FIELD, "junit");
+ addedViaJunit = true;
+ ++totalJunitAdded;
+ } if (i % allGroupsFactor == 0) {
+ doc.addField(AUTH_FIELD, allGroupsToken);
+ ++totalAllGroupsAdded;
+ if (!addedViaJunit) ++totalOnlyAllGroupsAdded;
+ }
+ docs.add(doc);
+ }
+ // make sure our factors give us interesting results --
+ // that some docs only have all groups and some only have junit
+ assert(totalOnlyAllGroupsAdded > 0);
+ assert(totalJunitAdded > totalAllGroupsAdded);
+
+ CloudSolrServer server = getCloudSolrServer(collectionName);
+ try {
+ server.add(docs);
+ server.commit(true, true);
+
+ // queries
+ SolrQuery query = new SolrQuery();
+ query.setQuery("*:*");
+
+ // as admin -- should only get all groups token documents
+ setAuthenticationUser("admin");
+ QueryResponse rsp = server.query(query);
+ SolrDocumentList docList = rsp.getResults();
+ assertEquals(totalAllGroupsAdded, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ assertEquals(0, Long.valueOf(id) % allGroupsFactor);
+ }
+
+ // as junit -- should get junit added + onlyAllGroupsAdded
+ setAuthenticationUser("junit");
+ rsp = server.query(query);
+ docList = rsp.getResults();
+ assertEquals(totalJunitAdded + totalOnlyAllGroupsAdded, docList.getNumFound());
+ for (SolrDocument doc : docList) {
+ String id = doc.getFieldValue("id").toString();
+ boolean addedJunit = (Long.valueOf(id) % junitFactor) == 0;
+ boolean onlyAllGroups = !addedJunit && (Long.valueOf(id) % allGroupsFactor) == 0;
+ assertEquals(true, addedJunit || onlyAllGroups);
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-sentry/blob/b4c39a1e/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
index 2aa68dd..66449ff 100644
--- a/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
+++ b/sentry-tests/sentry-tests-solr/src/test/resources/solr/collection1/conf/schema.xml
@@ -215,6 +215,8 @@
<dynamicField name="*_pi" type="pint" indexed="true" stored="true"/>
<dynamicField name="*_c" type="currency" indexed="true" stored="true"/>
+ <dynamicField name="*_auth" type="string" indexed="true" stored="false" multiValued="true"/>
+
<dynamicField name="ignored_*" type="ignored" multiValued="true"/>
<dynamicField name="attr_*" type="text_general" indexed="true" stored="true" multiValued="true"/>