You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by kkhatua <gi...@git.apache.org> on 2017/12/21 07:43:04 UTC
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles table
GitHub user kkhatua opened a pull request:
https://github.com/apache/drill/pull/1077
DRILL-5068: Create sys.profiles table
(Thanks to Hongze Zhang for the initial work)
Introduced a non-distributed (i.e. the foreman Drillbit executes a single fragment query) system table to list the completed profiles that are visible to it in the persistent store. This allows for querying the profiles on the following common metrics:
queryId
startTime
foreman
fragments
user
queue
planTime
queueTime
executeTime
totalTime
state
query
An additional system table - `sys.profiles_json` has also been introduced. This only has the following fields:
queryId
json
This allows for more detail metrics to be queried on the profile as a JSON document. Users are expected to use `sys.profiles` to identify profiles on the available metrics (such as long executeTime or fragments), and use the queryId to find the corresponding JSON document representation for further drill down into the profiles.
Here is what a sample output looks like:
```
[root@kk127 ~]# /opt/drill/apache-drill-1.13.0/bin/sqlline -u "jdbc:drill:drillbit=`hostname -i`"
...
apache drill 1.13.0-SNAPSHOT
"just drill it"
0: jdbc:drill:drillbit=10.10.100.127> select * from sys.profiles where startTime < date '2017-12-18' limit 10;
+---------------------------------------+--------------------------+---------------+------------+------------+--------+-----------+------------+--------------+------------+------------+--------------------------------------------------------+
| queryId | startTime | foreman | fragments | user | queue | planTime | queueTime | executeTime | totalTime | state | query |
+---------------------------------------+--------------------------+---------------+------------+------------+--------+-----------+------------+--------------+------------+------------+--------------------------------------------------------+
| 25cdcede-410d-6399-97f9-00ec2048f247 | 2017-12-14 08:06:57.062 | kk127.qa.lab | 1 | anonymous | - | 35 | 0 | 7718 | 7753 | COMPLETED | select queryId from profiles limit 1 |
| 25cdd1a2-7928-bcf9-a8cd-8e0fa653e2ce | 2017-12-14 07:55:09.551 | kk127.qa.lab | 1 | anonymous | - | 41 | 0 | 7580 | 7621 | COMPLETED | select queryId from profiles limit 1 |
| 25cdd1b6-3a56-e2b3-a6a1-4d1006ec8fac | 2017-12-14 07:54:49.403 | kk127.qa.lab | 1 | anonymous | - | 77 | 0 | 7452 | 7529 | COMPLETED | select queryId from profiles limit 1 |
| 25cdd1bd-963f-7a57-6a2d-cf7a3eac3f00 | 2017-12-14 07:54:41.065 | kk127.qa.lab | 0 | anonymous | - | 0 | 0 | 0 | 0 | FAILED | select profileJson from profiles limit 1 |
| 25cdd1e5-de23-0cfe-5c01-86621b31d8d5 | 2017-12-14 07:54:01.576 | kk127.qa.lab | 1 | anonymous | - | 1184 | 0 | 10860 | 12044 | COMPLETED | select profileJson from profiles_json limit 1 |
| 25cdd1e7-7c3a-5a25-0860-a86246bbc9ed | 2017-12-14 07:54:00.186 | kk127.qa.lab | 1 | anonymous | - | 102 | 0 | 231 | 333 | COMPLETED | use sys |
| 25cdd1f0-4473-55aa-a5eb-c85194321118 | 2017-12-14 07:53:51.966 | kk127.qa.lab | 0 | anonymous | - | 0 | 0 | 0 | 0 | FAILED | select profileJson from profiles_json limit 1 |
| 25cdd442-04cc-65fe-7a25-ac075fa22467 | 2017-12-14 07:43:57.251 | kk127.qa.lab | 1 | anonymous | - | 40 | 0 | 10553 | 10593 | COMPLETED | select profileJson from profiles_json limit 1 |
| 25cdd7c8-614d-8bcc-3b89-08df12761a3a | 2017-12-14 07:28:55.359 | kk127.qa.lab | 1 | anonymous | - | 170 | 0 | 10508 | 10678 | COMPLETED | select count(*) from (select * from profiles_json) |
| 25cdd7e0-34ea-9040-ab2a-ef8c6a136440 | 2017-12-14 07:28:31.464 | kk127.qa.lab | 0 | anonymous | - | 0 | 0 | 0 | 0 | FAILED | select count(*) from (pwdselect * from profiles_json) |
+---------------------------------------+--------------------------+---------------+------------+------------+--------+-----------+------------+--------------+------------+------------+--------------------------------------------------------+
10 rows selected (8.336 seconds)
0: jdbc:drill:drillbit=10.10.100.127> select * from sys.profiles_json where queryId = '25cdd1e7-7c3a-5a25-0860-a86246bbc9ed';
+---------+------+
| queryId | json |
+---------+------+
| 25cdd1e7-7c3a-5a25-0860-a86246bbc9ed | {"id":{"part1":2724064141780867621,"part2":603667490114619885},"type":1,"start":1513238040186,"end":1513238040519,"query":"use sys","foreman":{"address":"kk127.qa.lab","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0-SNAPSHOT","state":0},"state":2,"totalFragments":1,"finishedFragments":0,"fragmentProfile":[{"majorFragmentId":0,"minorFragmentProfile":[{"state":3,"minorFragmentId":0,"operatorProfile":[{"inputProfile":[{"records":1,"batches":2,"schemas":1}],"operatorId":0,"operatorType":26,"setupNanos":0,"processNanos":67070036,"peakLocalMemoryAllocated":4530688,"waitNanos":0},{"inputProfile":[{"records":0,"batches":0,"schemas":0}],"operatorId":0,"operatorType":13,"setupNanos":0,"processNanos":0,"peakLocalMemoryAllocated":4530176,"waitNanos":0},{"inputProfile":[{"records":1,"batches":1,"schemas":1}],"operatorId":0,"operatorType":13,"setupNanos":0,"processNanos":15787437,"peakLocalMemoryAllocated":4530176,"metric":[{"met
ricId":0,"longValue":41}],"waitNanos":18796208}],"startTime":1513238040329,"endTime":1513238040482,"memoryUsed":0,"maxMemoryUsed":9060864,"endpoint":{"address":"kk127.qa.lab","userPort":31010,"controlPort":31011,"dataPort":31012,"version":"1.12.0-SNAPSHOT","state":0},"lastUpdate":1513238040487,"lastProgress":1513238040487}]}],"user":"anonymous","optionsJson":"[ ]","planEnd":1513238040288,"queueWaitEnd":1513238040288,"totalCost":0.0,"queueName":"-"} |
+---------+------+
1 row selected (11.667 seconds)
```
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/kkhatua/drill DRILL-5068
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/drill/pull/1077.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #1077
----
commit 56f2cad756125923cd5f5e381ff17f5fd4ea5970
Author: Kunal Khatua <kk...@...>
Date: 2017-12-20T19:27:34Z
DRILL-5068: Create sys.profiles table
(Thanks to Hongze Zhang for the initial work)
Introduced a non-distributed (i.e. the foreman Drillbit executes a single fragment query) system table to list the completed profiles that are visible to it in the persistent store. This allows for querying the profiles on the following common metrics:
queryId
startTime
foreman
fragments
user
queue
planTime
queueTime
executeTime
totalTime
state
query
An additional system table - `sys.profiles_json` has also been introduced. This only has the following fields:
queryId
json
This allows for more detail metrics to be queried on the profile as a JSON document. Users are expected to use `sys.profiles` to identify profiles on the available metrics (such as long executeTime or fragments), and use the queryId to find the corresponding JSON document representation for further drill down into the profiles.
----
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159558355
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
--- End diff --
Got it. +1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159557228
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
--- End diff --
Legacy. I was originally trying to have the same iterator function for both JSON and tabular listing. +1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159288870
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
--- End diff --
Wildcard `<?>` is unnecessary since `itr` looks to be assigned by type `<ProfileInfo>` only
---
[GitHub] drill issue #1077: DRILL-5068: Create sys.profiles and sys.profiles_json tab...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on the issue:
https://github.com/apache/drill/pull/1077
Thanks for the changes. +1 LGTM.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159299747
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileJsonIterator.java ---
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles (JSON)
+ */
+public class ProfileJsonIterator implements Iterator<Object> {
--- End diff --
Lot of duplicate code between `ProfileJsonIterator` and `ProfileIterator`. You can make a base class `ProfileIterator` and 2 derived classes out of it `ProfileInfoIterator` and `ProfileJsonIterator`
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159289730
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
--- End diff --
Based on comment in FragmentContext.java, `QueryContext` won't be necessary here instead you can just pass in `QueryOptionsManager` as input which you can get from FragmentContext directly.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159560810
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
+ return Iterators.singletonIterator(ProfileInfo.getDefault());
+ }
+ }
+
+ //Returns an iterator for authorized profiles
+ private Iterator<Entry<String, QueryProfile>> getAuthorizedProfiles (
+ Iterator<Entry<String, QueryProfile>> allProfiles, String username, boolean isAdministrator) {
+ if (isAdministrator) {
+ return allProfiles;
+ }
+
+ List<Entry<String, QueryProfile>> authorizedProfiles = new LinkedList<Entry<String, QueryProfile>>();
+ while (allProfiles.hasNext()) {
+ Entry<String, QueryProfile> profileKVPair = allProfiles.next();
+ //Check if user matches
+ if (profileKVPair.getValue().getUser().equals(username)) {
+ authorizedProfiles.add(profileKVPair);
+ }
+ }
+ return authorizedProfiles.iterator();
+ }
+
+ /**
+ * Iterating persistentStore as a iterator of {@link org.apache.drill.exec.store.sys.ProfileIterator.ProfileInfo}.
+ */
+ private Iterator<ProfileInfo> transform(Iterator<Map.Entry<String, UserBitShared.QueryProfile>> all) {
--- End diff --
+1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159558552
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
--- End diff --
Fixed. +1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159289179
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
--- End diff --
is `QueryContext` ever expected to be `null` here ? If not then this should be `IllegalStateCheck` rather than returning false ?
---
[GitHub] drill issue #1077: DRILL-5068: Create sys.profiles and sys.profiles_json tab...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on the issue:
https://github.com/apache/drill/pull/1077
@sohami made the changes. You can review this.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159556423
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---
@@ -230,6 +230,10 @@ public DrillbitContext getDrillbitContext() {
return context;
}
+ public QueryContext getQueryContext() {
--- End diff --
I was using it to retrieve the user name via `queryContext.getQueryUserName()` but I can use the fragmentContext too. +1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159567549
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
+ return Iterators.singletonIterator(ProfileInfo.getDefault());
+ }
+ }
+
+ //Returns an iterator for authorized profiles
+ private Iterator<Entry<String, QueryProfile>> getAuthorizedProfiles (
+ Iterator<Entry<String, QueryProfile>> allProfiles, String username, boolean isAdministrator) {
+ if (isAdministrator) {
+ return allProfiles;
+ }
+
+ List<Entry<String, QueryProfile>> authorizedProfiles = new LinkedList<Entry<String, QueryProfile>>();
+ while (allProfiles.hasNext()) {
+ Entry<String, QueryProfile> profileKVPair = allProfiles.next();
+ //Check if user matches
+ if (profileKVPair.getValue().getUser().equals(username)) {
+ authorizedProfiles.add(profileKVPair);
+ }
+ }
+ return authorizedProfiles.iterator();
+ }
+
+ /**
+ * Iterating persistentStore as a iterator of {@link org.apache.drill.exec.store.sys.ProfileIterator.ProfileInfo}.
+ */
+ private Iterator<ProfileInfo> transform(Iterator<Map.Entry<String, UserBitShared.QueryProfile>> all) {
+ return Iterators.transform(all, new Function<Map.Entry<String, UserBitShared.QueryProfile>, ProfileInfo>() {
+ @Nullable
+ @Override
+ public ProfileInfo apply(@Nullable Map.Entry<String, UserBitShared.QueryProfile> input) {
+ if (input == null || input.getValue() == null) {
+ return ProfileInfo.getDefault();
+ }
+
+ //Constructing ProfileInfo
+ final String queryID = input.getKey();
+ final QueryProfile profile = input.getValue();
+ //For cases where query was never queued
+ final long assumedQueueEndTime = profile.getQueueWaitEnd()> 0 ? profile.getQueueWaitEnd() : profile.getPlanEnd();
+ return new ProfileInfo(
+ queryID,
+ new Timestamp(profile.getStart()),
+ profile.getForeman().getAddress(),
+ profile.getTotalFragments(),
+ profile.getUser(),
+ profile.getQueueName(),
+ computeDuration(profile.getStart(), profile.getPlanEnd()),
+ computeDuration(profile.getPlanEnd(), assumedQueueEndTime),
+ computeDuration(assumedQueueEndTime, profile.getEnd()),
+ profile.getState().name(),
+ profile.getQuery()
+ );
+ }
+ });
+ }
+
+ protected long computeDuration(long startTime, long endTime) {
+ if (endTime > startTime && startTime > 0) {
+ return (endTime - startTime);
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return itr.hasNext();
+ }
+
+ @Override
+ public Object next() {
+ return itr.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static class ProfileInfo {
+ private static final String UnknownValue = "UNKNOWN";
--- End diff --
+1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159298839
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
+ return Iterators.singletonIterator(ProfileInfo.getDefault());
+ }
+ }
+
+ //Returns an iterator for authorized profiles
+ private Iterator<Entry<String, QueryProfile>> getAuthorizedProfiles (
+ Iterator<Entry<String, QueryProfile>> allProfiles, String username, boolean isAdministrator) {
+ if (isAdministrator) {
+ return allProfiles;
+ }
+
+ List<Entry<String, QueryProfile>> authorizedProfiles = new LinkedList<Entry<String, QueryProfile>>();
+ while (allProfiles.hasNext()) {
+ Entry<String, QueryProfile> profileKVPair = allProfiles.next();
+ //Check if user matches
+ if (profileKVPair.getValue().getUser().equals(username)) {
+ authorizedProfiles.add(profileKVPair);
+ }
+ }
+ return authorizedProfiles.iterator();
+ }
+
+ /**
+ * Iterating persistentStore as a iterator of {@link org.apache.drill.exec.store.sys.ProfileIterator.ProfileInfo}.
+ */
+ private Iterator<ProfileInfo> transform(Iterator<Map.Entry<String, UserBitShared.QueryProfile>> all) {
--- End diff --
nit: Please change `Map.Entry --> Entry` here and below.
---
[GitHub] drill issue #1077: DRILL-5068: Create sys.profiles and sys.profiles_json tab...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on the issue:
https://github.com/apache/drill/pull/1077
@sohami
Applied the checks for security to deny unauthorized access.
This is a screenshot with an admin user (`mapr`) querying the table.
<img width="1440" alt="adminuser" src="https://user-images.githubusercontent.com/4335237/34444695-cbe7efd4-ec84-11e7-9bf1-e4b007c8a062.png">
This is a screenshot with a non-admin user (`kuser1`) querying the table.
<img width="1440" alt="nonadminuser" src="https://user-images.githubusercontent.com/4335237/34444715-f7f0b85e-ec84-11e7-8fe1-bbed71e2d5ea.png">
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/drill/pull/1077
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159567582
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
+ return Iterators.singletonIterator(ProfileInfo.getDefault());
+ }
+ }
+
+ //Returns an iterator for authorized profiles
+ private Iterator<Entry<String, QueryProfile>> getAuthorizedProfiles (
+ Iterator<Entry<String, QueryProfile>> allProfiles, String username, boolean isAdministrator) {
--- End diff --
+1 . Moved the authorizedProfiles method to base class.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159323362
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
--- End diff --
input parameter `context` is not used anywhere.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159558877
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
--- End diff --
I was trying to format the error message to indicate the issue., but I realize I might be hiding the actual error message. +1
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159557937
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
--- End diff --
Shouldn't be required any more.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159302253
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
+ return Iterators.singletonIterator(ProfileInfo.getDefault());
+ }
+ }
+
+ //Returns an iterator for authorized profiles
+ private Iterator<Entry<String, QueryProfile>> getAuthorizedProfiles (
+ Iterator<Entry<String, QueryProfile>> allProfiles, String username, boolean isAdministrator) {
+ if (isAdministrator) {
+ return allProfiles;
+ }
+
+ List<Entry<String, QueryProfile>> authorizedProfiles = new LinkedList<Entry<String, QueryProfile>>();
+ while (allProfiles.hasNext()) {
+ Entry<String, QueryProfile> profileKVPair = allProfiles.next();
+ //Check if user matches
+ if (profileKVPair.getValue().getUser().equals(username)) {
+ authorizedProfiles.add(profileKVPair);
+ }
+ }
+ return authorizedProfiles.iterator();
+ }
+
+ /**
+ * Iterating persistentStore as a iterator of {@link org.apache.drill.exec.store.sys.ProfileIterator.ProfileInfo}.
+ */
+ private Iterator<ProfileInfo> transform(Iterator<Map.Entry<String, UserBitShared.QueryProfile>> all) {
+ return Iterators.transform(all, new Function<Map.Entry<String, UserBitShared.QueryProfile>, ProfileInfo>() {
+ @Nullable
+ @Override
+ public ProfileInfo apply(@Nullable Map.Entry<String, UserBitShared.QueryProfile> input) {
+ if (input == null || input.getValue() == null) {
+ return ProfileInfo.getDefault();
+ }
+
+ //Constructing ProfileInfo
+ final String queryID = input.getKey();
+ final QueryProfile profile = input.getValue();
+ //For cases where query was never queued
+ final long assumedQueueEndTime = profile.getQueueWaitEnd()> 0 ? profile.getQueueWaitEnd() : profile.getPlanEnd();
+ return new ProfileInfo(
+ queryID,
+ new Timestamp(profile.getStart()),
+ profile.getForeman().getAddress(),
+ profile.getTotalFragments(),
+ profile.getUser(),
+ profile.getQueueName(),
+ computeDuration(profile.getStart(), profile.getPlanEnd()),
+ computeDuration(profile.getPlanEnd(), assumedQueueEndTime),
+ computeDuration(assumedQueueEndTime, profile.getEnd()),
+ profile.getState().name(),
+ profile.getQuery()
+ );
+ }
+ });
+ }
+
+ protected long computeDuration(long startTime, long endTime) {
+ if (endTime > startTime && startTime > 0) {
+ return (endTime - startTime);
+ } else {
+ return 0;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return itr.hasNext();
+ }
+
+ @Override
+ public Object next() {
+ return itr.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static class ProfileInfo {
+ private static final String UnknownValue = "UNKNOWN";
--- End diff --
nit: How about changing "UNKNOWN" to "N/A" since there are too many columns and a row with all UNKNOWN value might go to next line.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159323575
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
--- End diff --
`Exception e` should be an argument to `logger.error` not `String.format`
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159290765
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileIterator.java ---
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles
+ */
+public class ProfileIterator implements Iterator<Object> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ProfileIterator.class);
+
+ private final QueryProfileStoreContext profileStoreContext;
+ private final Iterator<?> itr;
+ private final String queryingUsername;
+ private final boolean isAdmin;
+
+ public ProfileIterator(FragmentContext context) {
+ //Grab profile Store Context
+ profileStoreContext = context
+ .getDrillbitContext()
+ .getProfileStoreContext();
+
+ queryingUsername = context.getQueryUserName();
+ isAdmin = hasAdminPrivileges(context.getQueryContext());
+
+ itr = iterateProfileInfo(context);
+ }
+
+ private boolean hasAdminPrivileges(QueryContext queryContext) {
+ if (queryContext == null) {
+ return false;
+ }
+ QueryOptionManager options = queryContext.getOptions();
+ if (queryContext.isUserAuthenticationEnabled() &&
+ !ImpersonationUtil.hasAdminPrivileges(
+ queryContext.getQueryUserName(),
+ ExecConstants.ADMIN_USERS_VALIDATOR.getAdminUsers(options),
+ ExecConstants.ADMIN_USER_GROUPS_VALIDATOR.getAdminUserGroups(options))) {
+ return false;
+ }
+
+ //Passed checks
+ return true;
+ }
+
+ private Iterator<ProfileInfo> iterateProfileInfo(FragmentContext context) {
+ try {
+ PersistentStore<UserBitShared.QueryProfile> profiles = profileStoreContext.getCompletedProfileStore();
+
+ return transform(getAuthorizedProfiles(profiles.getAll(), queryingUsername, isAdmin));
+
+ } catch (Exception e) {
+ logger.error(String.format("Unable to get persistence store: %s, ", profileStoreContext.getProfileStoreConfig().getName(), e));
+ return Iterators.singletonIterator(ProfileInfo.getDefault());
+ }
+ }
+
+ //Returns an iterator for authorized profiles
+ private Iterator<Entry<String, QueryProfile>> getAuthorizedProfiles (
+ Iterator<Entry<String, QueryProfile>> allProfiles, String username, boolean isAdministrator) {
--- End diff --
Feel's like methods `iterateProfileInfo` and `getAuthorizedProfiles` can be combined to a single method. All the input parameter's to this method can be directly accessed.
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by sohami <gi...@git.apache.org>.
Github user sohami commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159111473
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---
@@ -230,6 +230,10 @@ public DrillbitContext getDrillbitContext() {
return context;
}
+ public QueryContext getQueryContext() {
--- End diff --
How about instead having a public method like below as compared to returning entire QueryContext ?
```
public boolean isUserAuthenticationEnabled() {
return queryContext.isUserAuthenticationEnabled();
}
```
---
[GitHub] drill pull request #1077: DRILL-5068: Create sys.profiles and sys.profiles_j...
Posted by kkhatua <gi...@git.apache.org>.
Github user kkhatua commented on a diff in the pull request:
https://github.com/apache/drill/pull/1077#discussion_r159567509
--- Diff: exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileJsonIterator.java ---
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.sys;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.apache.drill.exec.server.QueryProfileStoreContext;
+import org.apache.drill.exec.server.options.QueryOptionManager;
+import org.apache.drill.exec.util.ImpersonationUtil;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * DRILL-5068: Add a new system table for completed profiles (JSON)
+ */
+public class ProfileJsonIterator implements Iterator<Object> {
--- End diff --
Reworked the code and tested. +1
---