You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2016/04/07 17:00:13 UTC
[38/50] [abbrv] incubator-asterixdb git commit: Move merged files
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-external-data/src/test/resources/classad-with-temporals.classads
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/resources/classad-with-temporals.classads b/asterixdb/asterix-external-data/src/test/resources/classad-with-temporals.classads
new file mode 100644
index 0000000..e20be09
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/resources/classad-with-temporals.classads
@@ -0,0 +1,134 @@
+
+ [
+ Schedd = "submit-5.chtc.wisc.edu";
+ BlockWrites = 3;
+ LastJobStatus = 2;
+ JobCurrentStartExecutingDate = 1459300924;
+ WantRemoteIO = true;
+ RequestCpus = 1;
+ NumShadowStarts = 1;
+ RemoteUserCpu = 6.607100000000000E+04;
+ NiceUser = false;
+ BytesRecvd = 7.292000000000000E+03;
+ RequestMemory = 12288;
+ ResidentSetSize = 750000;
+ StreamOut = false;
+ SpooledOutputFiles = "job697_results.tar.gz";
+ Arguments = "";
+ OnExitRemove = true;
+ ImageSize_RAW = 607024;
+ RemoteWallClockTime = 6.629100000000000E+04;
+ MachineAttrSlotWeight0 = 1;
+ ExecutableSize = 4;
+ JobStatus = 4;
+ DAGParentNodeNames = "";
+ ExitCode = 0;
+ DAGManNodesMask = "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27";
+ BytesSent = 8.580547200000000E+07;
+ LastRejMatchReason = "no match found ";
+ LastSuspensionTime = 0;
+ ExecutableSize_RAW = 4;
+ RecentBlockReadKbytes = 0;
+ TransferInputSizeMB = 0;
+ BlockReadKbytes = 0;
+ LocalSysCpu = 0.0;
+ Iwd = "/home/grandaduarte/mars/mhb1";
+ Cmd = "/home/grandaduarte/mars/mhb1/job697.sh";
+ CommittedSuspensionTime = 0;
+ RecentStatsLifetimeStarter = 1200;
+ TargetType = "Machine";
+ WhenToTransferOutput = "ON_EXIT";
+ BufferSize = 524288;
+ JobCurrentStartTransferOutputDate = 1459367212;
+ RecentBlockWrites = 0;
+ CompletionDate = 1459367213;
+ LastMatchTime = 1459300922;
+ LastJobLeaseRenewal = 1459367213;
+ DAGManNodesLog = "/home/grandaduarte/mars/mhb1/./dagman.dag.nodes.log";
+ ClusterId = 16798777;
+ JobUniverse = 5;
+ NumJobStarts = 1;
+ ProcId = 0;
+ PeriodicHold = false;
+ CondorPlatform = "$CondorPlatform: x86_64_RedHat6 $";
+ JobFinishedHookDone = 1459367213;
+ In = "/dev/null";
+ DiskUsage = 7500000;
+ EncryptExecuteDirectory = false;
+ User = "grandaduarte@chtc.wisc.edu";
+ LeaveJobInQueue = false;
+ Requirements = ( MY.JobUniverse == 12 || MY.JobUniverse == 7 || ( ( MY.WantFlocking || MY.WantGlidein || TARGET.PoolName == "CHTC" ) && ( TARGET.OpSysMajorVer == MY.LinuxVer || TARGET.OpSysMajorVer == MY.LinuxVerAlt || TARGET.OpSysMajorVer == MY.WinVer ) ) ) && ( TARGET.Arch == "X86_64" ) && ( TARGET.OpSys == "LINUX" ) && ( TARGET.Disk >= RequestDisk ) && ( TARGET.Memory >= RequestMemory ) && ( TARGET.HasFileTransfer );
+ MinHosts = 1;
+ MaxHosts = 1;
+ StartdPrincipal = "execute-side@matchsession/128.105.245.175";
+ JOBGLIDEIN_ResourceName = "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])";
+ EnteredCurrentStatus = 1459367213;
+ JobLeaseDuration = 2400;
+ QDate = 1459298672;
+ AccountingGroup = EngrPhysics_Wilson;
+ MemoryUsage = ( ( ResidentSetSize + 1023 ) / 1024 );
+ TerminationPending = true;
+ NumRestarts = 0;
+ NumSystemHolds = 0;
+ Environment = "";
+ LinuxVer = 6;
+ DAGNodeName = "_mars_MH1B1_661.inp";
+ CoreSize = 0;
+ OnExitHold = false;
+ CondorVersion = "$CondorVersion: 8.5.3 Mar 14 2016 BuildID: 358989 $";
+ UserLog = "/home/grandaduarte/mars/mhb1/job697.log";
+ JobCurrentStartDate = 1459300922;
+ MATCH_EXP_JOBGLIDEIN_ResourceName = "wisc.edu";
+ BufferBlockSize = 32768;
+ BlockWriteKbytes = 24;
+ ExitBySignal = false;
+ DAGManJobId = 16795779;
+ MachineAttrCpus0 = 1;
+ WantRemoteSyscalls = false;
+ CumulativeSuspensionTime = 0;
+ MyType = "Job";
+ Rank = 0.0;
+ JobNotification = 0;
+ Owner = "grandaduarte";
+ LinuxVerAlt = 6;
+ Err = "job697.err";
+ PeriodicRemove = false;
+ CommittedTime = 66291;
+ RecentBlockWriteKbytes = 0;
+ TransferIn = false;
+ ExitStatus = 0;
+ ShouldTransferFiles = "YES";
+ IsCHTCSubmit = true;
+ NumJobMatches = 1;
+ RootDir = "/";
+ JobStartDate = 1459300922;
+ JobPrio = 0;
+ CurrentHosts = 0;
+ GlobalJobId = "submit-5.chtc.wisc.edu#16798777.0#1459298672";
+ RemoteSysCpu = 6.100000000000000E+01;
+ LastRejMatchTime = 1459300921;
+ TotalSuspensions = 0;
+ CommittedSlotTime = 6.629100000000000E+04;
+ WantCheckpoint = false;
+ BlockReads = 0;
+ LastRemoteHost = "slot1_7@e375.chtc.wisc.edu";
+ TransferInput = "job697.sh";
+ LocalUserCpu = 0.0;
+ PeriodicRelease = false;
+ WinVer = 601;
+ LastPublicClaimId = "<128.105.245.175:9618>#1457031418#19008#...";
+ NumCkpts_RAW = 0;
+ Out = "job697.out";
+ SubmitEventNotes = "DAG Node: _mars_MH1B1_661.inp";
+ CumulativeSlotTime = 6.629100000000000E+04;
+ JobRunCount = 1;
+ RecentBlockReads = 0;
+ StreamErr = false;
+ DiskUsage_RAW = 6625678;
+ RequestDisk = 20971520;
+ ResidentSetSize_RAW = 597536;
+ OrigMaxHosts = 1;
+ NumCkpts = 0;
+ StatsLifetimeStarter = 66289;
+ ImageSize = 750000
+ ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-external-data/src/test/resources/results/classad-with-temporals.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/test/resources/results/classad-with-temporals.adm b/asterixdb/asterix-external-data/src/test/resources/results/classad-with-temporals.adm
new file mode 100644
index 0000000..3cd630b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/resources/results/classad-with-temporals.adm
@@ -0,0 +1 @@
+{ "GlobalJobId": "submit-5.chtc.wisc.edu#16798777.0#1459298672", "Owner": "grandaduarte", "ClusterId": 16798777i32, "ProcId": 0i32, "RemoteWallClockTime": duration("PT18H24M51S"), "CompletionDate": datetime("2016-03-30T19:46:53.000Z"), "QDate": datetime("2016-03-30T00:44:32.000Z"), "JobCurrentStartDate": datetime("2016-03-30T01:22:02.000Z"), "JobStartDate": datetime("2016-03-30T01:22:02.000Z"), "JobCurrentStartExecutingDate": datetime("2016-03-30T01:22:04.000Z"), "StatsLifetimeStarter": 66289, "SubmitEventNotes": "DAG Node: _mars_MH1B1_661.inp", "JobStatus": 4, "LeaveJobInQueue": false, "StartdPrincipal": "execute-side@matchsession/128.105.245.175", "OnExitRemove": true, "CoreSize": 0, "MATCH_EXP_JOBGLIDEIN_ResourceName": "wisc.edu", "Rank": 0.0d, "ExitStatus": 0, "ResidentSetSize": 750000, "WantCheckpoint": false, "In": "/dev/null", "MaxHosts": 1, "RootDir": "/", "NumRestarts": 0, "RecentBlockWriteKbytes": 0, "DiskUsage_RAW": 6625678, "EnteredCurrentStatus": 1459367213, "ResidentSe
tSize_RAW": 597536, "RequestDisk": 20971520, "MyType": "Job", "PeriodicRemove": false, "Cmd": "/home/grandaduarte/mars/mhb1/job697.sh", "CondorVersion": "$CondorVersion: 8.5.3 Mar 14 2016 BuildID: 358989 $", "ShouldTransferFiles": "YES", "TargetType": "Machine", "MinHosts": 1, "NumCkpts_RAW": 0, "RequestCpus": 1, "WinVer": 601, "RemoteUserCpu": 66071.0d, "BlockWrites": 3, "NiceUser": false, "Out": "job697.out", "ImageSize_RAW": 607024, "BytesSent": 8.5805472E7d, "CumulativeSuspensionTime": 0, "TransferIn": false, "NumCkpts": 0, "Err": "job697.err", "RecentBlockWrites": 0, "JobFinishedHookDone": 1459367213, "ImageSize": 750000, "Schedd": "submit-5.chtc.wisc.edu", "JobUniverse": 5, "EncryptExecuteDirectory": false, "TransferInputSizeMB": 0, "RecentBlockReads": 0, "SpooledOutputFiles": "job697_results.tar.gz", "BlockWriteKbytes": 24, "WhenToTransferOutput": "ON_EXIT", "ExitBySignal": false, "LastMatchTime": 1459300922, "OnExitHold": false, "OrigMaxHosts": 1, "RequestMemory": 12288, "Nu
mJobStarts": 1, "TerminationPending": true, "TotalSuspensions": 0, "BlockReads": 0, "DAGManJobId": 16795779, "MemoryUsage": "( ( ResidentSetSize + 1023 ) / 1024 )", "ExitCode": 0, "JobNotification": 0, "BlockReadKbytes": 0, "AccountingGroup": "EngrPhysics_Wilson", "NumJobMatches": 1, "LocalUserCpu": 0.0d, "LastJobStatus": 2, "BufferBlockSize": 32768, "CommittedTime": 66291, "ExecutableSize_RAW": 4, "LastRejMatchReason": "no match found ", "LastSuspensionTime": 0, "UserLog": "/home/grandaduarte/mars/mhb1/job697.log", "DAGManNodesMask": "0,1,2,4,5,7,9,10,11,12,13,16,17,24,27", "CumulativeSlotTime": 66291.0d, "LastJobLeaseRenewal": 1459367213, "MachineAttrSlotWeight0": 1, "NumSystemHolds": 0, "BytesRecvd": 7292.0d, "CondorPlatform": "$CondorPlatform: x86_64_RedHat6 $", "JOBGLIDEIN_ResourceName": "$$([IfThenElse(IsUndefined(TARGET.GLIDEIN_ResourceName), IfThenElse(IsUndefined(TARGET.GLIDEIN_Site), \"wisc.edu\", TARGET.GLIDEIN_Site), TARGET.GLIDEIN_ResourceName)])", "DAGNodeName": "_mars
_MH1B1_661.inp", "PeriodicRelease": false, "JobRunCount": 1, "LastRemoteHost": "slot1_7@e375.chtc.wisc.edu", "JobPrio": 0, "LocalSysCpu": 0.0d, "ExecutableSize": 4, "RemoteSysCpu": 61.0d, "TransferInput": "job697.sh", "PeriodicHold": false, "WantRemoteIO": true, "CommittedSuspensionTime": 0, "DAGParentNodeNames": "", "StreamErr": false, "RecentBlockReadKbytes": 0, "WantRemoteSyscalls": false, "NumShadowStarts": 1, "MachineAttrCpus0": 1, "DAGManNodesLog": "/home/grandaduarte/mars/mhb1/./dagman.dag.nodes.log", "Requirements": "( MY.JobUniverse == 12 || MY.JobUniverse == 7 || ( ( MY.WantFlocking || MY.WantGlidein || TARGET.PoolName == \"CHTC\" ) && ( TARGET.OpSysMajorVer == MY.LinuxVer || TARGET.OpSysMajorVer == MY.LinuxVerAlt || TARGET.OpSysMajorVer == MY.WinVer ) ) ) && ( TARGET.Arch == \"X86_64\" ) && ( TARGET.OpSys == \"LINUX\" ) && ( TARGET.Disk >= RequestDisk ) && ( TARGET.Memory >= RequestMemory ) && ( TARGET.HasFileTransfer )", "LinuxVerAlt": 6, "DiskUsage": 7500000, "LinuxVer"
: 6, "LastRejMatchTime": 1459300921, "JobLeaseDuration": 2400, "BufferSize": 524288, "IsCHTCSubmit": true, "JobCurrentStartTransferOutputDate": 1459367212, "RecentStatsLifetimeStarter": 1200, "CommittedSlotTime": 66291.0d, "Environment": "", "LastPublicClaimId": "<128.105.245.175:9618>#1457031418#19008#...", "Iwd": "/home/grandaduarte/mars/mhb1", "CurrentHosts": 0, "Arguments": "", "User": "grandaduarte@chtc.wisc.edu", "StreamOut": false }
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.1.script.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.1.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.1.script.aql
new file mode 100644
index 0000000..cc46136
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.1.script.aql
@@ -0,0 +1 @@
+create_and_start.sh
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.2.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.2.ddl.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.2.ddl.aql
new file mode 100644
index 0000000..d3317e4
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.2.ddl.aql
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a change feed with meta-data and test ingestion of records
+ * Expected Res : Success
+ * Date : 24th Feb 2016
+ */
+
+drop dataverse KeyVerse if exists;
+create dataverse KeyVerse;
+use dataverse KeyVerse;
+
+create type DocumentType as open{
+};
+
+create type KVMetaType as open{
+"key":string,
+bucket:string,
+vbucket:int32,
+seq:int64,
+cas:int64,
+creationTime:int64,
+expiration:int32,
+flags:int32,
+revSeq:int64,
+lockTime:int32
+};
+
+create dataset KVStore(DocumentType) with meta(KVMetaType)primary key meta()."key";
+
+create feed KVChangeStream using adapter(
+ ("type-name"="DocumentType"),
+ ("meta-type-name"="KVMetaType"),
+ ("reader"="kv_test"),
+ ("parser"="record-with-metadata"),
+ ("format"="dcp"),
+ ("record-format"="json"),
+ ("change-feed"="true"),
+ ("key-indexes"="0"),
+ ("key-indicators"="1"),
+ ("num-of-records"="1000")
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.3.update.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.3.update.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.3.update.aql
new file mode 100644
index 0000000..7faf013
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.3.update.aql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a change feed with meta-data and test ingestion of records
+ * Expected Res : Success
+ * Date : 24th Feb 2016
+ */
+use dataverse KeyVerse;
+
+set wait-for-completion-feed "true";
+connect feed KVChangeStream to dataset KVStore;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.4.script.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.4.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.4.script.aql
new file mode 100644
index 0000000..3ba1dc0
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.4.script.aql
@@ -0,0 +1 @@
+stop_and_start.sh
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.query.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.query.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.query.aql
new file mode 100644
index 0000000..9db20a9
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.query.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Description : Create a change feed and test ingestion of records
+ * Expected Res : Success
+ * Date : 24th Feb 2016
+ */
+use dataverse KeyVerse;
+
+count(
+ for $d in dataset KVStore
+ return $d
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.6.script.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.6.script.aql b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.6.script.aql
new file mode 100644
index 0000000..10e1a51
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/queries/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.6.script.aql
@@ -0,0 +1 @@
+stop_and_delete.sh
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.adm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.adm b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.adm
new file mode 100644
index 0000000..c31da8b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/results/query_after_restart/dataset-with-meta-record/dataset-with-meta-record.5.adm
@@ -0,0 +1 @@
+804
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/create_and_start.sh
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/create_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/create_and_start.sh
new file mode 100755
index 0000000..945f01d
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/create_and_start.sh
@@ -0,0 +1 @@
+$MANAGIX_HOME/bin/managix create -n nc1 -c $MANAGIX_HOME/clusters/local/local.xml;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_delete.sh
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_delete.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_delete.sh
new file mode 100755
index 0000000..d7deea3
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_delete.sh
@@ -0,0 +1,3 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix delete -n nc1;
+
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_start.sh
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_start.sh b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_start.sh
new file mode 100755
index 0000000..1271a2b
--- /dev/null
+++ b/asterixdb/asterix-installer/src/test/resources/transactionts/scripts/query_after_restart/dataset-with-meta-record/stop_and_start.sh
@@ -0,0 +1,2 @@
+$MANAGIX_HOME/bin/managix stop -n nc1;
+$MANAGIX_HOME/bin/managix start -n nc1;
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineColumnAliasVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineColumnAliasVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineColumnAliasVisitor.java
new file mode 100644
index 0000000..bb8c149
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/InlineColumnAliasVisitor.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.base.Literal;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.clause.LimitClause;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.clause.WhereClause;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
+import org.apache.asterix.lang.common.expression.FieldBinding;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.IfExpr;
+import org.apache.asterix.lang.common.expression.IndexAccessor;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.OperatorExpr;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.UnaryExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.struct.QuantifiedPair;
+import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.HavingClause;
+import org.apache.asterix.lang.sqlpp.clause.JoinClause;
+import org.apache.asterix.lang.sqlpp.clause.NestClause;
+import org.apache.asterix.lang.sqlpp.clause.Projection;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
+import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableSubstitutionUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor;
+
+public class InlineColumnAliasVisitor extends AbstractSqlppQueryExpressionVisitor<Void, Boolean> {
+
+ private final ScopeChecker scopeChecker = new ScopeChecker();
+ private final LangRewritingContext context;
+
+ public InlineColumnAliasVisitor(LangRewritingContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Void visit(WhereClause whereClause, Boolean arg) throws AsterixException {
+ whereClause.getWhereExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(FromClause fromClause, Boolean arg) throws AsterixException {
+ for (FromTerm fromTerm : fromClause.getFromTerms()) {
+ fromTerm.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(FromTerm fromTerm, Boolean arg) throws AsterixException {
+ fromTerm.getLeftExpression().accept(this, arg);
+ // A from binding variable will override the alias to substitute.
+ scopeChecker.getCurrentScope().removeSymbolExpressionMapping(fromTerm.getLeftVariable());
+ if (fromTerm.hasPositionalVariable()) {
+ scopeChecker.getCurrentScope().removeSymbolExpressionMapping(fromTerm.getPositionalVariable());
+ }
+
+ for (AbstractBinaryCorrelateClause correlate : fromTerm.getCorrelateClauses()) {
+ correlate.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(JoinClause joinClause, Boolean arg) throws AsterixException {
+ joinClause.getRightExpression().accept(this, arg);
+ removeSubsutitions(joinClause);
+ joinClause.getConditionExpression().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(NestClause nestClause, Boolean arg) throws AsterixException {
+ nestClause.getRightExpression().accept(this, arg);
+ nestClause.getConditionExpression().accept(this, arg);
+ removeSubsutitions(nestClause);
+ return null;
+ }
+
+ @Override
+ public Void visit(UnnestClause unnestClause, Boolean arg) throws AsterixException {
+ unnestClause.getRightExpression().accept(this, arg);
+ removeSubsutitions(unnestClause);
+ return null;
+ }
+
+ @Override
+ public Void visit(Projection projection, Boolean arg) throws AsterixException {
+ projection.getExpression().accept(this, arg);
+ VariableExpr columnAlias = new VariableExpr(
+ SqlppVariableUtil.toInternalVariableIdentifier(projection.getName()));
+ VariableSubstitutionEnvironment env = scopeChecker.getCurrentScope().getVarSubstitutionEnvironment();
+ Expression gbyKey = (Expression) SqlppRewriteUtil.deepCopy(env.findSubstituion(columnAlias));
+ if (arg) {
+ scopeChecker.getCurrentScope().addSymbolExpressionMappingToScope(columnAlias, projection.getExpression());
+ } else {
+ if (gbyKey != null) {
+ projection.setExpression(gbyKey);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(SelectBlock selectBlock, Boolean arg) throws AsterixException {
+ // Traverses the select block in the order of "select", "group-by",
+ // "group-by" lets and "having".
+ selectBlock.getSelectClause().accept(this, true);
+
+ if (selectBlock.hasFromClause()) {
+ selectBlock.getFromClause().accept(this, arg);
+ }
+ if (selectBlock.hasLetClauses()) {
+ for (LetClause letClause : selectBlock.getLetList()) {
+ letClause.accept(this, arg);
+ }
+ }
+ if (selectBlock.hasGroupbyClause()) {
+ selectBlock.getGroupbyClause().accept(this, arg);
+ }
+ if (selectBlock.hasLetClausesAfterGroupby()) {
+ for (LetClause letClauseAfterGby : selectBlock.getLetListAfterGroupby()) {
+ letClauseAfterGby.accept(this, true);
+ }
+ }
+ if (selectBlock.hasHavingClause()) {
+ selectBlock.getHavingClause().accept(this, arg);
+ }
+
+ // Visit select clause again to overwrite projection expressions if the group-by clause is rewritten.
+ selectBlock.getSelectClause().accept(this, false);
+ return null;
+ }
+
+ @Override
+ public Void visit(SelectClause selectClause, Boolean arg) throws AsterixException {
+ if (selectClause.selectElement()) {
+ selectClause.getSelectElement().accept(this, arg);
+ }
+ if (selectClause.selectRegular()) {
+ selectClause.getSelectRegular().accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(SelectElement selectElement, Boolean arg) throws AsterixException {
+ Expression expr = selectElement.getExpression();
+ expr.accept(this, arg);
+ if (expr.getKind() == Kind.RECORD_CONSTRUCTOR_EXPRESSION) {
+ // To be consistent with SelectRegular.
+ mapForRecordConstructor(arg, (RecordConstructor) expr);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(SelectRegular selectRegular, Boolean arg) throws AsterixException {
+ for (Projection projection : selectRegular.getProjections()) {
+ projection.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(SelectSetOperation selectSetOperation, Boolean arg) throws AsterixException {
+ selectSetOperation.getLeftInput().accept(this, arg);
+ for (SetOperationRight right : selectSetOperation.getRightInputs()) {
+ right.getSetOperationRightInput().accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(SelectExpression selectExpression, Boolean arg) throws AsterixException {
+ scopeChecker.createNewScope();
+
+ // Visits let bindings.
+ if (selectExpression.hasLetClauses()) {
+ for (LetClause lc : selectExpression.getLetList()) {
+ lc.accept(this, arg);
+ }
+ }
+
+ // Visits selectSetOperation.
+ selectExpression.getSelectSetOperation().accept(this, arg);
+
+ // Visits order by.
+ if (selectExpression.hasOrderby()) {
+ selectExpression.getOrderbyClause().accept(this, arg);
+ }
+
+ // Visits limit.
+ if (selectExpression.hasLimit()) {
+ selectExpression.getLimitClause().accept(this, arg);
+ }
+
+ // Exits the scope that were entered within this select expression
+ scopeChecker.removeCurrentScope();
+ return null;
+ }
+
+ @Override
+ public Void visit(LetClause letClause, Boolean rewrite) throws AsterixException {
+ VariableSubstitutionEnvironment env = scopeChecker.getCurrentScope().getVarSubstitutionEnvironment();
+ if (rewrite) {
+ Expression newBindExpr = (Expression) SqlppVariableSubstitutionUtil
+ .substituteVariableWithoutContext(letClause.getBindingExpr(), env);
+ letClause.setBindingExpr(newBindExpr);
+ }
+ letClause.getBindingExpr().accept(this, false);
+ // A let binding variable will override the alias to substitute.
+ scopeChecker.getCurrentScope().removeSymbolExpressionMapping(letClause.getVarExpr());
+ return null;
+ }
+
+ @Override
+ public Void visit(OrderbyClause oc, Boolean arg) throws AsterixException {
+ VariableSubstitutionEnvironment env = scopeChecker.getCurrentScope().getVarSubstitutionEnvironment();
+ List<Expression> orderExprs = new ArrayList<Expression>();
+ for (Expression orderExpr : oc.getOrderbyList()) {
+ orderExprs.add((Expression) SqlppVariableSubstitutionUtil.substituteVariableWithoutContext(orderExpr, env));
+ orderExpr.accept(this, arg);
+ }
+ oc.setOrderbyList(orderExprs);
+ return null;
+ }
+
+ @Override
+ public Void visit(GroupbyClause gc, Boolean arg) throws AsterixException {
+ VariableSubstitutionEnvironment env = scopeChecker.getCurrentScope().getVarSubstitutionEnvironment();
+ Map<VariableExpr, VariableExpr> oldGbyExprsToNewGbyVarMap = new HashMap<>();
+ for (GbyVariableExpressionPair gbyVarExpr : gc.getGbyPairList()) {
+ Expression oldGbyExpr = gbyVarExpr.getExpr();
+ Expression newExpr = (Expression) SqlppVariableSubstitutionUtil.substituteVariableWithoutContext(oldGbyExpr,
+ env);
+ newExpr.accept(this, arg);
+ gbyVarExpr.setExpr(newExpr);
+ if (gbyVarExpr.getVar() == null) {
+ gbyVarExpr.setVar(new VariableExpr(context.newVariable()));
+ }
+ if (oldGbyExpr.getKind() == Kind.VARIABLE_EXPRESSION) {
+ VariableExpr oldGbyVarExpr = (VariableExpr) oldGbyExpr;
+ if (env.findSubstituion(oldGbyVarExpr) != null) {
+ // Re-mapping that needs to be added.
+ oldGbyExprsToNewGbyVarMap.put(oldGbyVarExpr, gbyVarExpr.getVar());
+ }
+ }
+ }
+ for (Entry<VariableExpr, VariableExpr> entry : oldGbyExprsToNewGbyVarMap.entrySet()) {
+ // The group-by key variable will override the alias to substitute.
+ scopeChecker.getCurrentScope().removeSymbolExpressionMapping(entry.getKey());
+ scopeChecker.getCurrentScope().addSymbolExpressionMappingToScope(entry.getKey(), entry.getValue());
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(LimitClause limitClause, Boolean arg) throws AsterixException {
+ limitClause.getLimitExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(HavingClause havingClause, Boolean arg) throws AsterixException {
+ VariableSubstitutionEnvironment env = scopeChecker.getCurrentScope().getVarSubstitutionEnvironment();
+ Expression newFilterExpr = (Expression) SqlppVariableSubstitutionUtil
+ .substituteVariableWithoutContext(havingClause.getFilterExpression(), env);
+ newFilterExpr.accept(this, arg);
+ havingClause.setFilterExpression(newFilterExpr);
+ return null;
+ }
+
+ @Override
+ public Void visit(Query q, Boolean arg) throws AsterixException {
+ q.getBody().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(FunctionDecl fd, Boolean arg) throws AsterixException {
+ scopeChecker.createNewScope();
+ fd.getFuncBody().accept(this, arg);
+ scopeChecker.removeCurrentScope();
+ return null;
+ }
+
+ @Override
+ public Void visit(LiteralExpr l, Boolean arg) throws AsterixException {
+ return null;
+ }
+
+ @Override
+ public Void visit(ListConstructor lc, Boolean arg) throws AsterixException {
+ for (Expression expr : lc.getExprList()) {
+ expr.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(RecordConstructor rc, Boolean rewrite) throws AsterixException {
+ for (FieldBinding binding : rc.getFbList()) {
+ binding.getLeftExpr().accept(this, false);
+ binding.getRightExpr().accept(this, false);
+ }
+ return null;
+ }
+
+ private void mapForRecordConstructor(Boolean initPhase, RecordConstructor rc) throws AsterixException {
+ for (FieldBinding binding : rc.getFbList()) {
+ Expression leftExpr = binding.getLeftExpr();
+ if (leftExpr.getKind() == Kind.LITERAL_EXPRESSION) {
+ LiteralExpr literalExpr = (LiteralExpr) leftExpr;
+ if (literalExpr.getValue().getLiteralType() == Literal.Type.STRING) {
+ String fieldName = literalExpr.getValue().getStringValue();
+ VariableExpr columnAlias = new VariableExpr(
+ SqlppVariableUtil.toInternalVariableIdentifier(fieldName));
+ VariableSubstitutionEnvironment env = scopeChecker.getCurrentScope()
+ .getVarSubstitutionEnvironment();
+ if (initPhase) {
+ scopeChecker.getCurrentScope().addSymbolExpressionMappingToScope(columnAlias,
+ binding.getRightExpr());
+ } else {
+ Expression gbyKey = (Expression) SqlppRewriteUtil.deepCopy(env.findSubstituion(columnAlias));
+ if (gbyKey != null) {
+ binding.setRightExpr(gbyKey);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public Void visit(OperatorExpr operatorExpr, Boolean arg) throws AsterixException {
+ for (Expression expr : operatorExpr.getExprList()) {
+ expr.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(IfExpr ifExpr, Boolean arg) throws AsterixException {
+ ifExpr.getCondExpr().accept(this, arg);
+ ifExpr.getThenExpr().accept(this, arg);
+ ifExpr.getElseExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(QuantifiedExpression qe, Boolean arg) throws AsterixException {
+ for (QuantifiedPair pair : qe.getQuantifiedList()) {
+ pair.getExpr().accept(this, arg);
+ }
+ qe.getSatisfiesExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(CallExpr callExpr, Boolean arg) throws AsterixException {
+ for (Expression expr : callExpr.getExprList()) {
+ expr.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(VariableExpr varExpr, Boolean arg) throws AsterixException {
+ return null;
+ }
+
+ @Override
+ public Void visit(UnaryExpr u, Boolean arg) throws AsterixException {
+ u.getExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(FieldAccessor fa, Boolean arg) throws AsterixException {
+ fa.getExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(IndexAccessor ia, Boolean arg) throws AsterixException {
+ ia.getExpr().accept(this, arg);
+ Expression indexExpr = ia.getExpr();
+ if (indexExpr != null) {
+ indexExpr.accept(this, arg);
+ }
+ return null;
+ }
+
+ private void removeSubsutitions(AbstractBinaryCorrelateClause unnestClause) {
+ scopeChecker.getCurrentScope().removeSymbolExpressionMapping(unnestClause.getRightVariable());
+ if (unnestClause.hasPositionalVariable()) {
+ scopeChecker.getCurrentScope().removeSymbolExpressionMapping(unnestClause.getPositionalVariable());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
new file mode 100644
index 0000000..c7c7d11
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
+
+public class SqlppBuiltinFunctionRewriteVisitor extends AbstractSqlppSimpleExpressionVisitor {
+
+ @Override
+ public Expression visit(CallExpr callExpr, Expression arg) throws AsterixException {
+ //TODO(buyingyi): rewrite SQL temporal functions
+ FunctionSignature functionSignature = callExpr.getFunctionSignature();
+ callExpr.setFunctionSignature(FunctionMapUtil.normalizeBuiltinFunctionSignature(functionSignature, true));
+ List<Expression> newExprList = new ArrayList<Expression>();
+ for (Expression expr : callExpr.getExprList()) {
+ newExprList.add(expr.accept(this, arg));
+ }
+ callExpr.setExprList(newExprList);
+ return callExpr;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java
new file mode 100644
index 0000000..ae629af
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGlobalAggregationSugarVisitor.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.literal.IntegerLiteral;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.visitor.CheckSql92AggregateVisitor;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
+
+public class SqlppGlobalAggregationSugarVisitor extends AbstractSqlppSimpleExpressionVisitor {
+
+ @Override
+ public Expression visit(SelectBlock selectBlock, Expression arg) throws AsterixException {
+ SelectClause selectClause = selectBlock.getSelectClause();
+ if (!selectBlock.hasGroupbyClause() && selectBlock.hasFromClause()) {
+ boolean addImplicitGby = false;
+ if (selectClause.selectRegular()) {
+ addImplicitGby = isSql92Aggregate(selectClause.getSelectRegular(), selectBlock);
+ } else {
+ addImplicitGby = isSql92Aggregate(selectClause.getSelectElement(), selectBlock);
+ }
+ if (addImplicitGby) {
+ // Adds an implicit group-by clause for SQL-92 global aggregate.
+ List<GbyVariableExpressionPair> gbyPairList = new ArrayList<>();
+ gbyPairList.add(new GbyVariableExpressionPair(null, new LiteralExpr(new IntegerLiteral(1))));
+ List<GbyVariableExpressionPair> decorPairList = new ArrayList<>();
+ List<VariableExpr> withVarList = new ArrayList<>();
+ GroupbyClause gbyClause = new GroupbyClause(gbyPairList, decorPairList, withVarList, null, null, false,
+ true);
+ selectBlock.setGroupbyClause(gbyClause);
+ }
+ }
+ return super.visit(selectBlock, arg);
+ }
+
+ private boolean isSql92Aggregate(ILangExpression expr, SelectBlock selectBlock) throws AsterixException {
+ CheckSql92AggregateVisitor visitor = new CheckSql92AggregateVisitor();
+ return expr.accept(visitor, selectBlock);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupBySugarVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupBySugarVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupBySugarVisitor.java
new file mode 100644
index 0000000..ae47264
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupBySugarVisitor.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
+import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableSubstitutionUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor;
+
+/**
+ * An AST pre-processor to rewrite group-by sugar queries.
+ */
+public class SqlppGroupBySugarVisitor extends AbstractSqlppExpressionScopingVisitor {
+
+ private final Expression groupVar;
+ private final Collection<VariableExpr> targetVars;
+
+ public SqlppGroupBySugarVisitor(LangRewritingContext context, Expression groupVar,
+ Collection<VariableExpr> targetVars) {
+ super(context);
+ this.groupVar = groupVar;
+ this.targetVars = targetVars;
+ }
+
+ @Override
+ public Expression visit(CallExpr callExpr, Expression arg) throws AsterixException {
+ List<Expression> newExprList = new ArrayList<Expression>();
+ FunctionSignature signature = callExpr.getFunctionSignature();
+ boolean aggregate = FunctionMapUtil.isSql92AggregateFunction(signature)
+ || FunctionMapUtil.isCoreAggregateFunction(signature);
+ boolean rewritten = false;
+ for (Expression expr : callExpr.getExprList()) {
+ Expression newExpr = aggregate ? wrapAggregationArgument(expr) : expr;
+ rewritten |= newExpr != expr;
+ newExprList.add(newExpr.accept(this, arg));
+ }
+ if (rewritten) {
+ // Rewrites the SQL-92 function name to core functions.
+ callExpr.setFunctionSignature(FunctionMapUtil.sql92ToCoreAggregateFunction(signature));
+ }
+ callExpr.setExprList(newExprList);
+ return callExpr;
+ }
+
+ private Expression wrapAggregationArgument(Expression expr) throws AsterixException {
+ if (expr.getKind() == Kind.SELECT_EXPRESSION) {
+ return expr;
+ }
+ Set<VariableExpr> definedVars = scopeChecker.getCurrentScope().getLiveVariables();
+ Set<VariableExpr> vars = new HashSet<>(targetVars);
+ vars.remove(definedVars); // Exclude re-defined local variables.
+ Set<VariableExpr> freeVars = SqlppRewriteUtil.getFreeVariable(expr);
+ if (!vars.containsAll(freeVars)) {
+ return expr;
+ }
+
+ VariableExpr var = new VariableExpr(context.newVariable());
+ FromTerm fromTerm = new FromTerm(groupVar, var, null, null);
+ FromClause fromClause = new FromClause(Collections.singletonList(fromTerm));
+
+ // Select clause.
+ SelectElement selectElement = new SelectElement(expr);
+ SelectClause selectClause = new SelectClause(selectElement, null, false);
+
+ // Construct the select expression.
+ SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, null, null, null, null, null);
+ SelectSetOperation selectSetOperation = new SelectSetOperation(new SetOperationInput(selectBlock, null), null);
+ SelectExpression selectExpression = new SelectExpression(null, selectSetOperation, null, null, false);
+ selectExpression.setSubquery(true);
+
+ // replace variable expressions with field access
+ Map<VariableExpr, Expression> varExprMap = new HashMap<>();
+ for (VariableExpr usedVar : freeVars) {
+ varExprMap.put(usedVar,
+ new FieldAccessor(var, SqlppVariableUtil.toUserDefinedVariableName(usedVar.getVar())));
+ }
+ selectElement.setExpression(
+ (Expression) SqlppVariableSubstitutionUtil.substituteVariableWithoutContext(expr, varExprMap));
+ return selectExpression;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java
new file mode 100644
index 0000000..c9e7a6e
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppGroupByVisitor.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.context.Scope;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+/**
+ * A pre-processor that adds the group variable as well as its group field
+ * list into the AST. It will also invoke SQL group-by aggregation sugar rewritings.
+ */
+public class SqlppGroupByVisitor extends AbstractSqlppExpressionScopingVisitor {
+
+ public SqlppGroupByVisitor(LangRewritingContext context) {
+ super(context);
+ }
+
+ @Override
+ public Expression visit(SelectBlock selectBlock, Expression arg) throws AsterixException {
+ // Traverses the select block in the order of "from", "let"s, "where",
+ // "group by", "let"s, "having" and "select".
+ if (selectBlock.hasFromClause()) {
+ selectBlock.getFromClause().accept(this, arg);
+ }
+ if (selectBlock.hasLetClauses()) {
+ List<LetClause> letList = selectBlock.getLetList();
+ for (LetClause letClause : letList) {
+ letClause.accept(this, arg);
+ }
+ }
+ if (selectBlock.hasWhereClause()) {
+ selectBlock.getWhereClause().accept(this, arg);
+ }
+ if (selectBlock.hasGroupbyClause()) {
+ selectBlock.getGroupbyClause().accept(this, arg);
+ Set<VariableExpr> withVarSet = new HashSet<>(selectBlock.getGroupbyClause().getWithVarList());
+ withVarSet.remove(selectBlock.getGroupbyClause().getGroupVar());
+ if (selectBlock.hasLetClausesAfterGroupby()) {
+ List<LetClause> letListAfterGby = selectBlock.getLetListAfterGroupby();
+ for (LetClause letClauseAfterGby : letListAfterGby) {
+ // Rewrites each let clause after the group-by.
+ SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(selectBlock.getGroupbyClause().getGroupVar(),
+ withVarSet, letClauseAfterGby, context);
+ letClauseAfterGby.accept(this, arg);
+ }
+ }
+ if (selectBlock.hasHavingClause()) {
+ // Rewrites the having clause.
+ SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(selectBlock.getGroupbyClause().getGroupVar(),
+ withVarSet, selectBlock.getHavingClause(), context);
+ selectBlock.getHavingClause().accept(this, arg);
+ }
+ // Rewrites the select clause.
+ SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(selectBlock.getGroupbyClause().getGroupVar(),
+ withVarSet, selectBlock.getSelectClause(), context);
+
+ SelectExpression parentSelectExpression = (SelectExpression) arg;
+ if (parentSelectExpression.hasOrderby()) {
+ // Rewrites the order-by clause.
+ SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(selectBlock.getGroupbyClause().getGroupVar(),
+ withVarSet, parentSelectExpression.getOrderbyClause(), context);
+ }
+ if (parentSelectExpression.hasLimit()) {
+ // Rewrites the limit clause.
+ SqlppRewriteUtil.rewriteExpressionUsingGroupVariable(selectBlock.getGroupbyClause().getGroupVar(),
+ withVarSet, parentSelectExpression.getLimitClause(), context);
+ }
+ }
+ selectBlock.getSelectClause().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Expression visit(GroupbyClause gc, Expression arg) throws AsterixException {
+ Scope newScope = scopeChecker.extendCurrentScopeNoPush(true);
+ // Puts all group-by variables into the symbol set of the new scope.
+ for (GbyVariableExpressionPair gbyVarExpr : gc.getGbyPairList()) {
+ gbyVarExpr.setExpr(gbyVarExpr.getExpr().accept(this, arg));
+ VariableExpr gbyVar = gbyVarExpr.getVar();
+ if (gbyVar != null) {
+ newScope.addNewVarSymbolToScope(gbyVarExpr.getVar().getVar());
+ }
+ }
+ // Puts all live variables into withVarList.
+ List<VariableExpr> withVarList = new ArrayList<VariableExpr>();
+ Iterator<Identifier> varIterator = scopeChecker.getCurrentScope().liveSymbols();
+ while (varIterator.hasNext()) {
+ Identifier ident = varIterator.next();
+ VariableExpr varExpr = new VariableExpr();
+ if (ident instanceof VarIdentifier) {
+ varExpr.setIsNewVar(false);
+ varExpr.setVar((VarIdentifier) ident);
+ withVarList.add(varExpr);
+ newScope.addNewVarSymbolToScope((VarIdentifier) ident);
+ }
+ }
+
+ // Sets the field list for the group variable.
+ List<Pair<Expression, Identifier>> groupFieldList = new ArrayList<>();
+ if (!gc.hasGroupFieldList()) {
+ for (VariableExpr varExpr : withVarList) {
+ Pair<Expression, Identifier> varIdPair = new Pair<>(new VariableExpr(varExpr.getVar()),
+ SqlppVariableUtil.toUserDefinedVariableName(varExpr.getVar()));
+ groupFieldList.add(varIdPair);
+ }
+ gc.setGroupFieldList(groupFieldList);
+ } else {
+ // Check the scopes of group field variables.
+ for (Pair<Expression, Identifier> groupField : gc.getGroupFieldList()) {
+ Expression newVar = groupField.first.accept(this, arg);
+ groupFieldList.add(new Pair<>(newVar, groupField.second));
+ }
+ }
+ gc.setGroupFieldList(groupFieldList);
+
+ // Sets the group variable.
+ if (!gc.hasGroupVar()) {
+ VariableExpr groupVar = new VariableExpr(context.newVariable());
+ gc.setGroupVar(groupVar);
+ }
+ newScope.addNewVarSymbolToScope(gc.getGroupVar().getVar());
+
+ // Adds the group variable into the "with" (i.e., re-binding) variable list.
+ VariableExpr gbyVarRef = new VariableExpr(gc.getGroupVar().getVar());
+ gbyVarRef.setIsNewVar(false);
+ withVarList.add(gbyVarRef);
+ gc.setWithVarList(withVarList);
+
+ scopeChecker.replaceCurrentScope(newScope);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java
new file mode 100644
index 0000000..e7832bb
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppInlineUdfsVisitor.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IRewriterFactory;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.visitor.AbstractInlineUdfsVisitor;
+import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.HavingClause;
+import org.apache.asterix.lang.sqlpp.clause.JoinClause;
+import org.apache.asterix.lang.sqlpp.clause.NestClause;
+import org.apache.asterix.lang.sqlpp.clause.Projection;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableSubstitutionUtil;
+import org.apache.asterix.lang.sqlpp.visitor.SqlppCloneAndSubstituteVariablesVisitor;
+import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class SqlppInlineUdfsVisitor extends AbstractInlineUdfsVisitor
+ implements ISqlppVisitor<Boolean, List<FunctionDecl>> {
+
+ /**
+ * @param context,
+ * manages ids of variables and guarantees uniqueness of variables.
+ * @param rewriterFactory,
+ * a rewrite factory for rewriting user-defined functions.
+ * @param declaredFunctions,
+ * a list of declared functions associated with the query.
+ * @param metadataProvider,
+ * providing the definition of created (i.e., stored) user-defined functions.
+ */
+ public SqlppInlineUdfsVisitor(LangRewritingContext context, IRewriterFactory rewriterFactory,
+ List<FunctionDecl> declaredFunctions, AqlMetadataProvider metadataProvider) {
+ super(context, rewriterFactory, declaredFunctions, metadataProvider,
+ new SqlppCloneAndSubstituteVariablesVisitor(context));
+ }
+
+ @Override
+ protected Expression generateQueryExpression(List<LetClause> letClauses, Expression returnExpr)
+ throws AsterixException {
+ Map<VariableExpr, Expression> varExprMap = extractLetBindingVariableExpressionMappings(letClauses);
+ Expression inlinedReturnExpr = (Expression) SqlppVariableSubstitutionUtil
+ .substituteVariableWithoutContext(returnExpr, varExprMap);
+ return inlinedReturnExpr;
+ }
+
+ @Override
+ public Boolean visit(FromClause fromClause, List<FunctionDecl> func) throws AsterixException {
+ boolean changed = false;
+ for (FromTerm fromTerm : fromClause.getFromTerms()) {
+ changed |= fromTerm.accept(this, func);
+ }
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(FromTerm fromTerm, List<FunctionDecl> func) throws AsterixException {
+ boolean changed = false;
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(fromTerm.getLeftExpression(), func);
+ fromTerm.setLeftExpression(p.second);
+ changed |= p.first;
+ for (AbstractBinaryCorrelateClause correlateClause : fromTerm.getCorrelateClauses()) {
+ changed |= correlateClause.accept(this, func);
+ }
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(JoinClause joinClause, List<FunctionDecl> funcs) throws AsterixException {
+ Pair<Boolean, Expression> p1 = inlineUdfsInExpr(joinClause.getRightExpression(), funcs);
+ joinClause.setRightExpression(p1.second);
+ Pair<Boolean, Expression> p2 = inlineUdfsInExpr(joinClause.getConditionExpression(), funcs);
+ joinClause.setConditionExpression(p2.second);
+ return p1.first || p2.first;
+ }
+
+ @Override
+ public Boolean visit(NestClause nestClause, List<FunctionDecl> funcs) throws AsterixException {
+ Pair<Boolean, Expression> p1 = inlineUdfsInExpr(nestClause.getRightExpression(), funcs);
+ nestClause.setRightExpression(p1.second);
+ Pair<Boolean, Expression> p2 = inlineUdfsInExpr(nestClause.getConditionExpression(), funcs);
+ nestClause.setConditionExpression(p2.second);
+ return p1.first || p2.first;
+ }
+
+ @Override
+ public Boolean visit(Projection projection, List<FunctionDecl> funcs) throws AsterixException {
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(projection.getExpression(), funcs);
+ projection.setExpression(p.second);
+ return p.first;
+ }
+
+ @Override
+ public Boolean visit(SelectBlock selectBlock, List<FunctionDecl> funcs) throws AsterixException {
+ boolean changed = false;
+ if (selectBlock.hasFromClause()) {
+ changed |= selectBlock.getFromClause().accept(this, funcs);
+ }
+ if (selectBlock.hasLetClauses()) {
+ for (LetClause letClause : selectBlock.getLetList()) {
+ changed |= letClause.accept(this, funcs);
+ }
+ }
+ if (selectBlock.hasWhereClause()) {
+ changed |= selectBlock.getWhereClause().accept(this, funcs);
+ }
+ if (selectBlock.hasGroupbyClause()) {
+ changed |= selectBlock.getGroupbyClause().accept(this, funcs);
+ }
+ if (selectBlock.hasLetClausesAfterGroupby()) {
+ for (LetClause letClause : selectBlock.getLetListAfterGroupby()) {
+ changed |= letClause.accept(this, funcs);
+ }
+ }
+ if (selectBlock.hasHavingClause()) {
+ changed |= selectBlock.getHavingClause().accept(this, funcs);
+ }
+ changed |= selectBlock.getSelectClause().accept(this, funcs);
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(SelectClause selectClause, List<FunctionDecl> funcs) throws AsterixException {
+ boolean changed = false;
+ if (selectClause.selectElement()) {
+ changed |= selectClause.getSelectElement().accept(this, funcs);
+ } else {
+ changed |= selectClause.getSelectRegular().accept(this, funcs);
+ }
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(SelectElement selectElement, List<FunctionDecl> funcs) throws AsterixException {
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(selectElement.getExpression(), funcs);
+ selectElement.setExpression(p.second);
+ return p.first;
+ }
+
+ @Override
+ public Boolean visit(SelectRegular selectRegular, List<FunctionDecl> funcs) throws AsterixException {
+ boolean changed = false;
+ for (Projection projection : selectRegular.getProjections()) {
+ changed |= projection.accept(this, funcs);
+ }
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(SelectSetOperation selectSetOperation, List<FunctionDecl> funcs) throws AsterixException {
+ boolean changed = false;
+ changed |= selectSetOperation.getLeftInput().accept(this, funcs);
+ for (SetOperationRight right : selectSetOperation.getRightInputs()) {
+ changed |= right.getSetOperationRightInput().accept(this, funcs);
+ }
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(SelectExpression selectExpression, List<FunctionDecl> funcs) throws AsterixException {
+ boolean changed = false;
+ if (selectExpression.hasLetClauses()) {
+ for (LetClause letClause : selectExpression.getLetList()) {
+ changed |= letClause.accept(this, funcs);
+ }
+ }
+ changed |= selectExpression.getSelectSetOperation().accept(this, funcs);
+ if (selectExpression.hasOrderby()) {
+ changed |= selectExpression.getOrderbyClause().accept(this, funcs);
+ }
+ if (selectExpression.hasLimit()) {
+ changed |= selectExpression.getLimitClause().accept(this, funcs);
+ }
+ return changed;
+ }
+
+ @Override
+ public Boolean visit(UnnestClause unnestClause, List<FunctionDecl> funcs) throws AsterixException {
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(unnestClause.getRightExpression(), funcs);
+ unnestClause.setRightExpression(p.second);
+ return p.first;
+ }
+
+ @Override
+ public Boolean visit(HavingClause havingClause, List<FunctionDecl> funcs) throws AsterixException {
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(havingClause.getFilterExpression(), funcs);
+ havingClause.setFilterExpression(p.second);
+ return p.first;
+ }
+
+ private Map<VariableExpr, Expression> extractLetBindingVariableExpressionMappings(List<LetClause> letClauses)
+ throws AsterixException {
+ Map<VariableExpr, Expression> varExprMap = new HashMap<VariableExpr, Expression>();
+ for (LetClause lc : letClauses) {
+ // inline let variables one by one iteratively.
+ lc.setBindingExpr((Expression) SqlppVariableSubstitutionUtil
+ .substituteVariableWithoutContext(lc.getBindingExpr(), varExprMap));
+ varExprMap.put(lc.getVarExpr(), lc.getBindingExpr());
+ }
+ return varExprMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/d630d1a2/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
new file mode 100644
index 0000000..5ca2533
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.config.MetadataConstants;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+
+public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopingVisitor {
+
+ protected final boolean overwrite;
+ protected final AqlMetadataProvider metadataProvider;
+
+ /**
+ * @param context,
+ * manages ids of variables and guarantees uniqueness of variables.
+ * @param overwrite,
+ * whether rewrite unbounded variables to dataset function calls.
+ * This flag can only be true for rewriting a top-level query.
+ * It should be false for rewriting the body expression of a user-defined function.
+ */
+ public VariableCheckAndRewriteVisitor(LangRewritingContext context, boolean overwrite,
+ AqlMetadataProvider metadataProvider) {
+ super(context);
+ this.overwrite = overwrite;
+ this.metadataProvider = metadataProvider;
+ }
+
+ @Override
+ public Expression visit(VariableExpr varExpr, Expression arg) throws AsterixException {
+ String varName = varExpr.getVar().getValue();
+ if (scopeChecker.isInForbiddenScopes(varName)) {
+ throw new AsterixException(
+ "Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
+ }
+ if (rewriteNeeded(varExpr)) {
+ return datasetRewrite(varExpr);
+ } else {
+ return varExpr;
+ }
+ }
+
+ // Whether a rewrite is needed for a variable reference expression.
+ private boolean rewriteNeeded(VariableExpr varExpr) throws AsterixException {
+ String varName = varExpr.getVar().getValue();
+ Identifier ident = scopeChecker.lookupSymbol(varName);
+ if (ident != null) {
+ // Exists such an identifier
+ varExpr.setIsNewVar(false);
+ varExpr.setVar((VarIdentifier) ident);
+ return false;
+ } else {
+ // Meets a undefined variable
+ return true;
+ }
+ }
+
+ // Rewrites for global variable (e.g., dataset) references.
+ private Expression datasetRewrite(VariableExpr expr) throws AsterixException {
+ if (!overwrite) {
+ return expr;
+ }
+ String funcName = "dataset";
+ String dataverse = MetadataConstants.METADATA_DATAVERSE_NAME;
+ FunctionSignature signature = new FunctionSignature(dataverse, funcName, 1);
+ List<Expression> argList = new ArrayList<Expression>();
+ //Ignore the parser-generated prefix "$" for a dataset.
+ String dataset = SqlppVariableUtil.toUserDefinedVariableName(expr.getVar()).getValue();
+ argList.add(new LiteralExpr(new StringLiteral(dataset)));
+ return new CallExpr(signature, argList);
+ }
+}