You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2016/06/29 14:38:05 UTC
[06/10] ambari git commit: Revert "AMBARI-17079. Moved Hue to Ambari
migrator to standard view architecture (Pradarttana Panda via dipayanb)"
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
deleted file mode 100644
index 9f9e053..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryMigrationUtility.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.ambari.view.huetoambarimigration.migration.hive.savedquery;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceAmbariDatabase;
-import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceHueDatabase;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.HiveModel;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationModel;
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.hive.savedqueryset.MysqlQuerySetAmbariDB;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.hive.savedqueryset.OracleQuerySetAmbariDB;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.hive.savedqueryset.PostgressQuerySetAmbariDB;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.hive.savedqueryset.QuerySetAmbariDB;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.hive.savedqueryset.*;
-import org.apache.ambari.view.huetoambarimigration.resources.PersonalCRUDResourceManager;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.MigrationResourceManager;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.apache.ambari.view.huetoambarimigration.migration.configuration.ConfigurationCheckImplementation;
-import org.apache.log4j.Logger;
-
-import java.beans.PropertyVetoException;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.text.ParseException;
-import java.util.ArrayList;
-
-public class HiveSavedQueryMigrationUtility {
-
-
-
- protected MigrationResourceManager resourceManager = null;
-
- public synchronized PersonalCRUDResourceManager<MigrationResponse> getResourceManager(ViewContext view) {
- if (resourceManager == null) {
- resourceManager = new MigrationResourceManager(view);
- }
- return resourceManager;
- }
-
- public MigrationModel hiveSavedQueryMigration(String username, String instance, String startDate, String endDate, ViewContext view, MigrationResponse migrationresult, String jobid) throws IOException, ItemNotFound {
-
- long startTime = System.currentTimeMillis();
-
- final Logger logger = Logger.getLogger(HiveSavedQueryMigrationUtility.class);
-
- Connection connectionAmbaridb = null;
- Connection connectionHuedb = null;
-
- int i = 0;
-
- logger.info("-------------------------------------");
- logger.info("hive saved query Migration started");
- logger.info("-------------------------------------");
- logger.info("start date: " + startDate);
- logger.info("enddate date: " + endDate);
- logger.info("instance is: " + instance);
- logger.info("hue username is : " + username);
-
- HiveSavedQueryMigrationImplementation hivesavedqueryimpl = new HiveSavedQueryMigrationImplementation();/* creating Implementation object */
-
- QuerySet huedatabase=null;
-
- if(view.getProperties().get("huedrivername").contains("mysql"))
- {
- huedatabase=new MysqlQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("postgresql"))
- {
- huedatabase=new PostgressQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("sqlite"))
- {
- huedatabase=new SqliteQuerySet();
- }
- else if (view.getProperties().get("huedrivername").contains("oracle"))
- {
- huedatabase=new OracleQuerySet();
- }
-
-
- QuerySetAmbariDB ambaridatabase=null;
-
-
- if(view.getProperties().get("ambaridrivername").contains("mysql"))
- {
- ambaridatabase=new MysqlQuerySetAmbariDB();
- }
- else if(view.getProperties().get("ambaridrivername").contains("postgresql"))
- {
- ambaridatabase=new PostgressQuerySetAmbariDB();
- }
- else if (view.getProperties().get("ambaridrivername").contains("oracle"))
- {
- ambaridatabase= new OracleQuerySetAmbariDB();
- }
-
- int maxcountForHivehistroryAmbaridb, maxCountforSavequeryAmbaridb;
- String time = null;
- Long epochtime = null;
- String dirNameforHiveSavedquery;
- ArrayList<HiveModel> dbpojoHiveSavedQuery = new ArrayList<HiveModel>();
-
- try {
-
- connectionHuedb = DataSourceHueDatabase.getInstance(view.getProperties().get("huedrivername"), view.getProperties().get("huejdbcurl"), view.getProperties().get("huedbusername"), view.getProperties().get("huedbpassword")).getConnection(); /* fetching connection to hue DB */
-
- dbpojoHiveSavedQuery = hivesavedqueryimpl.fetchFromHuedb(username, startDate, endDate, connectionHuedb,huedatabase); /* fetching data from hue db and storing it in to a model */
-
-
- for(int j=0;j<dbpojoHiveSavedQuery.size();j++)
- {
- logger.info("the query fetched from hue"+dbpojoHiveSavedQuery.get(j).getQuery());
-
- }
-
-
- if (dbpojoHiveSavedQuery.size() == 0) /* if no data has been fetched from hue db according to search criteria */ {
-
- migrationresult.setIsNoQuerySelected("yes");
- migrationresult.setProgressPercentage(0);
- migrationresult.setNumberOfQueryTransfered(0);
- migrationresult.setTotalNoQuery(dbpojoHiveSavedQuery.size());
- getResourceManager(view).update(migrationresult, jobid);
- logger.info("No queries has been selected acccording to your criteria");
-
- logger.info("no hive saved query has been selected from hue according to your criteria of searching");
-
-
- } else {
-
- connectionAmbaridb = DataSourceAmbariDatabase.getInstance(view.getProperties().get("ambaridrivername"), view.getProperties().get("ambarijdbcurl"), view.getProperties().get("ambaridbusername"), view.getProperties().get("ambaridbpassword")).getConnection();/* connecting to ambari DB */
- connectionAmbaridb.setAutoCommit(false);
-
- for (i = 0; i < dbpojoHiveSavedQuery.size(); i++) {
-
- logger.info("_____________________");
- logger.info("Loop No." + (i + 1));
- logger.info("_____________________");
-
- float calc = ((float) (i + 1)) / dbpojoHiveSavedQuery.size() * 100;
- int progressPercentage = Math.round(calc);
-
- migrationresult.setIsNoQuerySelected("no");
- migrationresult.setProgressPercentage(progressPercentage);
- migrationresult.setNumberOfQueryTransfered(i+1);
- migrationresult.setTotalNoQuery(dbpojoHiveSavedQuery.size());
- getResourceManager(view).update(migrationresult, jobid);
-
-
-
-
- logger.info("query fetched from hue:- " + dbpojoHiveSavedQuery.get(i).getQuery());
-
- int tableIdSavedQuery = hivesavedqueryimpl.fetchInstancetablenameForSavedqueryHive(connectionAmbaridb, instance,ambaridatabase); /* fetching the instance table name for migration saved query from the given instance name */
-
- int tableIdHistoryHive = hivesavedqueryimpl.fetchInstanceTablenameHiveHistory(connectionAmbaridb, instance,ambaridatabase); /* fetching the instance table name for migration history query from the given instance name */
-
- logger.info("Table name are fetched from instance name.");
-
- hivesavedqueryimpl.writetoFilequeryHql(dbpojoHiveSavedQuery.get(i).getQuery(), ConfigurationCheckImplementation.getHomeDir()); /* writing migration query to a local file*/
-
- hivesavedqueryimpl.writetoFileLogs(ConfigurationCheckImplementation.getHomeDir());/* writing logs to localfile */
-
- logger.info(".hql and logs file are saved in temporary directory");
-
- maxcountForHivehistroryAmbaridb = (hivesavedqueryimpl.fetchMaxdsidFromHiveHistory( connectionAmbaridb, tableIdHistoryHive,ambaridatabase) + 1);/* fetching the maximum ds_id from migration history table*/
-
- maxCountforSavequeryAmbaridb = (hivesavedqueryimpl.fetchMaxidforSavedQueryHive(connectionAmbaridb, tableIdSavedQuery,ambaridatabase) + 1);/* fetching the maximum ds_id from migration saved query table*/
-
- time = hivesavedqueryimpl.getTime();/* getting system time */
-
- epochtime = hivesavedqueryimpl.getEpochTime();/* getting epoch time */
-
- dirNameforHiveSavedquery = "/user/admin/migration/jobs/migration-job-" + maxcountForHivehistroryAmbaridb + "-"
- + time + "/"; // creating hdfs directory name
-
- logger.info("Directory will be creted in HDFS" + dirNameforHiveSavedquery);
-
- hivesavedqueryimpl.insertRowHiveHistory(dirNameforHiveSavedquery,maxcountForHivehistroryAmbaridb,epochtime,connectionAmbaridb,tableIdHistoryHive,instance,i,ambaridatabase);// inserting to migration history table
-
- logger.info("Row inserted in hive History table.");
-
- if (view.getProperties().get("KerberoseEnabled").equals("y")) {
-
- logger.info("Kerberose Enabled");
- hivesavedqueryimpl.createDirHiveSecured(dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// creating directory in hdfs in kerborized cluster
- hivesavedqueryimpl.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting .hql file in hdfs in kerberoroized cluster
- hivesavedqueryimpl.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting logs file in hdfs in kerberoroized cluster
-
- } else {
-
- logger.info("Kerberose Not Enabled");
- hivesavedqueryimpl.createDirHive(dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// creating directory in hdfs
- hivesavedqueryimpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "query.hql", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting .hql file in hdfs directory
- hivesavedqueryimpl.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + "logs", dirNameforHiveSavedquery, view.getProperties().get("namenode_URI_Ambari"));// putting logs file in hdfs
- }
-
- //inserting into hived saved query table
- //6.
- hivesavedqueryimpl.insertRowinSavedQuery(maxCountforSavequeryAmbaridb, dbpojoHiveSavedQuery.get(i).getDatabase(), dirNameforHiveSavedquery, dbpojoHiveSavedQuery.get(i).getQuery(), dbpojoHiveSavedQuery.get(i).getOwner(), connectionAmbaridb, tableIdSavedQuery, instance, i,ambaridatabase);
-
- }
- connectionAmbaridb.commit();
-
- }
-
-
- } catch (SQLException e) {
-
- logger.error("SQL exception: ", e);
- try {
- connectionAmbaridb.rollback();
- logger.info("roll back done");
- } catch (SQLException e1) {
- logger.error("Rollback error: ", e1);
-
- }
- } catch (ClassNotFoundException e1) {
- logger.error("Class not found : " , e1);
- } catch (ParseException e) {
- logger.error("ParseException: " , e);
- } catch (URISyntaxException e) {
- logger.error("URISyntaxException: " , e);
- } catch (PropertyVetoException e) {
- logger.error("PropertyVetoException:" , e);
- } finally {
- if (null != connectionAmbaridb)
- try {
- connectionAmbaridb.close();
- } catch (SQLException e) {
- logger.error("Error in connection close", e);
- }
- }
-
-
- hivesavedqueryimpl.deleteFileQueryhql(ConfigurationCheckImplementation.getHomeDir());
- hivesavedqueryimpl.deleteFileQueryLogs(ConfigurationCheckImplementation.getHomeDir());
-
- long stopTime = System.currentTimeMillis();
- long elapsedTime = stopTime - startTime;
-
- MigrationModel model=new MigrationModel();
-
- migrationresult.setJobtype("hivesavedquerymigration");
- migrationresult.setTotalTimeTaken(String.valueOf(elapsedTime));
- getResourceManager(view).update(migrationresult, jobid);
-
-
-
- logger.info("-------------------------------");
- logger.info("hive saved query Migration end");
- logger.info("--------------------------------");
-
- return model;
-
- }
-}
-
-
-
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
deleted file mode 100644
index 0445132..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/hive/savedquery/HiveSavedQueryStartJob.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.huetoambarimigration.migration.hive.savedquery;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.json.simple.JSONObject;
-
-import java.io.IOException;
-
-
-public class HiveSavedQueryStartJob extends Thread{
-
- String username;
- String instance;
- String startdate;
- String enddate;
- String jobid;
- ViewContext view;
-
- public HiveSavedQueryStartJob(String username, String instance, String startdate, String enddate, String jobid, ViewContext view) {
- this.username = username;
- this.instance=instance;
- this.startdate=startdate;
- this.enddate=enddate;
- this.jobid=jobid;
- this.view=view;
- }
-
-
-
- @Override
- public void run() {
-
- MigrationResponse migrationresult=new MigrationResponse();
-
- migrationresult.setId(jobid);
- migrationresult.setIntanceName(instance);
- migrationresult.setUserNameofhue(username);
- migrationresult.setProgressPercentage(0);
-
- JSONObject response = new JSONObject();
-
- /**
- * creating a separate thread
- */
-
- HiveSavedQueryMigrationUtility hivesavedquery=new HiveSavedQueryMigrationUtility();
- try {
- hivesavedquery.hiveSavedQueryMigration(username,instance,startdate,enddate,view,migrationresult,jobid);
- }
- catch (IOException e) {
- e.printStackTrace();
- } catch (ItemNotFound itemNotFound) {
- itemNotFound.printStackTrace();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
deleted file mode 100644
index 64e7069..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationImplementation.java
+++ /dev/null
@@ -1,532 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.view.huetoambarimigration.migration.pig.pigjob;
-
-import java.security.PrivilegedExceptionAction;
-import java.sql.*;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.GregorianCalendar;
-import java.io.*;
-import java.net.URISyntaxException;
-;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.pig.jobqueryset.QuerySet;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.jobqueryset.*;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.PigModel;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Logger;
-import org.jdom.Attribute;
-import org.jdom.Document;
-import org.jdom.Element;
-import org.jdom.JDOMException;
-import org.jdom.input.SAXBuilder;
-import org.jdom.output.Format;
-import org.jdom.output.XMLOutputter;
-
-import org.apache.ambari.view.huetoambarimigration.migration.configuration.ConfigurationCheckImplementation;
-
-public class PigJobMigrationImplementation {
-
- static final Logger logger = Logger.getLogger(PigJobMigrationImplementation.class);
-
- private static String readAll(Reader rd) throws IOException {
- StringBuilder sb = new StringBuilder();
- int cp;
- while ((cp = rd.read()) != -1) {
- sb.append((char) cp);
- }
- return sb.toString();
- }
-
- public void wrtitetoalternatesqlfile(String dirname, String content, String instance, int i) throws IOException {
- Date dNow = new Date();
- SimpleDateFormat ft = new SimpleDateFormat("YYYY-MM-dd hh:mm:ss");
- String currentDate = ft.format(dNow);
- XMLOutputter xmlOutput = new XMLOutputter();
- xmlOutput.setFormat(Format.getPrettyFormat());
- File xmlfile = new File(ConfigurationCheckImplementation.getHomeDir() + "RevertChangesService.xml");
- if (xmlfile.exists()) {
- String iteration = Integer.toString(i + 1);
- SAXBuilder builder = new SAXBuilder();
- Document doc;
- try {
- doc = (Document) builder.build(xmlfile);
- Element rootNode = doc.getRootElement();
- Element record = new Element("RevertRecord");
- record.setAttribute(new Attribute("id", iteration));
- record.addContent(new Element("datetime").setText(currentDate.toString()));
- record.addContent(new Element("dirname").setText(dirname));
- record.addContent(new Element("instance").setText(instance));
- record.addContent(new Element("query").setText(content));
- rootNode.addContent(record);
- xmlOutput.output(doc, new FileWriter(ConfigurationCheckImplementation.getHomeDir() + "RevertChangesService.xml"));
- } catch (JDOMException e) {
-
- logger.error("Jdom Exception: ", e);
- }
-
-
- } else {
- // create
- try {
- String iteration = Integer.toString(i + 1);
- Element revertrecord = new Element("RevertChangePage");
- Document doc = new Document(revertrecord);
- doc.setRootElement(revertrecord);
- Element record = new Element("RevertRecord");
- record.setAttribute(new Attribute("id", iteration));
- record.addContent(new Element("datetime").setText(currentDate.toString()));
- record.addContent(new Element("dirname").setText(dirname));
- record.addContent(new Element("instance").setText(instance));
- record.addContent(new Element("query").setText(content));
- doc.getRootElement().addContent(record);
- xmlOutput.output(doc, new FileWriter(ConfigurationCheckImplementation.getHomeDir() + "RevertChangesService.xml"));
- } catch (IOException io) {
- logger.error("Jdom Exception: ", io);
- }
-
- }
-
- }
-
- public int fetchMaxIdforPigJob(Connection c, int id, QuerySetAmbariDB ambaridatabase) throws SQLException {
-
-
- String ds_id = null;
- ResultSet rs = null;
- PreparedStatement prSt = null;
-
- prSt = ambaridatabase.getMaxDsIdFromTableId(c, id);
-
- rs = prSt.executeQuery();
-
- while (rs.next()) {
- ds_id = rs.getString("max");
- }
-
- int num;
- if (ds_id == null) {
- num = 1;
- } else {
- num = Integer.parseInt(ds_id);
- }
- return num;
-
- }
-
- public int fetchInstanceTablename(Connection c, String instance, QuerySetAmbariDB ambaridatabase) throws SQLException {
-
-
- String ds_id = new String();
- int id = 0;
- Statement stmt = null;
- PreparedStatement prSt = null;
-
-
- ResultSet rs = null;
-
-
- prSt = ambaridatabase.getTableIdFromInstanceName(c, instance);
-
- logger.info("sql statement to fetch is from ambari instance:= = " + prSt);
-
- rs = prSt.executeQuery();
-
- while (rs.next()) {
- id = rs.getInt("id");
- }
- return id;
- }
-
- public void insertRowPigJob(String dirname, int maxcountforpigjob, String time, String time2, long epochtime, String title, Connection c, int id, String status, String instance, int i, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
-
- String epochtime1 = Long.toString(epochtime);
- String maxcountforpigjob1 = Integer.toString(maxcountforpigjob);
- String ds_id = new String();
- String revSql;
-
- PreparedStatement prSt = null;
-
- prSt = ambaridatabase.insertToPigJob(dirname, maxcountforpigjob1, epochtime, title, c, id, status);
-
- prSt.executeUpdate();
-
- revSql = ambaridatabase.revertSql(id, maxcountforpigjob1);
-
- wrtitetoalternatesqlfile(dirname, revSql, instance, i);
-
- }
-
- public long getEpochTime() throws ParseException {
- int day, month, year;
- int second, minute, hour;
- int milisecond;
- GregorianCalendar date = new GregorianCalendar();
-
- day = date.get(Calendar.DAY_OF_MONTH);
- month = date.get(Calendar.MONTH);
- year = date.get(Calendar.YEAR);
-
- second = date.get(Calendar.SECOND);
- minute = date.get(Calendar.MINUTE);
- hour = date.get(Calendar.HOUR);
- milisecond = date.get(Calendar.MILLISECOND);
- String s1 = year + "-" + (month + 1) + "-" + day + "_" + hour + "-" + minute + "-" + second + "-" + milisecond;
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS");
- Date date1 = df.parse(s1);
- long epoch = date1.getTime();
- return epoch;
-
- }
-
- public String getTime() throws ParseException {
- int day, month, year;
- int second, minute, hour;
- int milisecond;
- GregorianCalendar date = new GregorianCalendar();
-
- day = date.get(Calendar.DAY_OF_MONTH);
- month = date.get(Calendar.MONTH);
- year = date.get(Calendar.YEAR);
-
- second = date.get(Calendar.SECOND);
- minute = date.get(Calendar.MINUTE);
- hour = date.get(Calendar.HOUR);
- milisecond = date.get(Calendar.MILLISECOND);
- String s = year + "-" + (month + 1) + "-" + day + "_" + hour + "-" + minute;
- String s1 = year + "-" + (month + 1) + "-" + day + "_" + hour + "-" + minute + "-" + second + "-" + milisecond;
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS");
- Date date1 = df.parse(s1);
- long epoch = date1.getTime();
- return s;
-
- }
-
- public String getTimeInorder() throws ParseException {
- SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.msssss +00:00:00");//dd/MM/yyyy
- Date now = new Date();
- String strDate = sdfDate.format(now);
- return strDate;
- }
-
- public ArrayList<PigModel> fetchFromHueDB(String username, String startdate, String endtime, Connection connection, QuerySet huedatabase) throws ClassNotFoundException, IOException {
- int id = 0;
- int i = 0;
- String[] query = new String[100];
- ArrayList<PigModel> pigjobarraylist = new ArrayList<PigModel>();
- try {
- connection.setAutoCommit(false);
- PreparedStatement prSt = null;
- Statement statement = connection.createStatement();
- ResultSet rs;
-
- ResultSet rs1 = null;
- if (username.equals("all")) {
- } else {
-
- prSt = huedatabase.getUseridfromUserName(connection, username);
-
- rs = prSt.executeQuery();
-
- while (rs.next()) {
- id = rs.getInt("id");
- }
- }
-
- if (startdate.equals("") && endtime.equals("")) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesNoStartDateNoEndDateAllUser(connection);
- } else {
- prSt = huedatabase.getQueriesNoStartDateNoEndDate(connection, id);
-
- }
-
- } else if ((startdate.equals("")) && !(endtime.equals(""))) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesNoStartDateYesEndDateAllUser(connection, endtime);
- } else {
- prSt = huedatabase.getQueriesNoStartDateYesEndDate(connection, id, endtime);
-
- }
- } else if (!(startdate.equals("")) && (endtime.equals(""))) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesYesStartDateNoEndDateAllUser(connection, startdate);
- } else {
- prSt = huedatabase.getQueriesYesStartDateNoEndDate(connection, id, startdate);
-
- }
-
- } else if (!(startdate.equals("")) && !(endtime.equals(""))) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesYesStartDateYesEndDateAllUser(connection, startdate, endtime);
- } else {
- prSt = huedatabase.getQueriesYesStartDateYesEndDate(connection, id, startdate, endtime);
- }
-
-
- }
-
- rs1 = prSt.executeQuery();
-
-
- while (rs1.next()) {
- PigModel pigjjobobject = new PigModel();
-
- int runstatus = rs1.getInt("status");
-
- if (runstatus == 1) {
- pigjjobobject.setStatus("RUNNING");
- } else if (runstatus == 2) {
- pigjjobobject.setStatus("SUCCEEDED");
- } else if (runstatus == 3) {
- pigjjobobject.setStatus("SUBMIT_FAILED");
- } else if (runstatus == 4) {
- pigjjobobject.setStatus("KILLED");
- }
- String title = rs1.getString("script_title");
-
-
- pigjjobobject.setTitle(title);
- String dir = rs1.getString("statusdir");
- pigjjobobject.setDir(dir);
- Date created_data = rs1.getDate("start_time");
- pigjjobobject.setDt(created_data);
-
- pigjobarraylist.add(pigjjobobject);
-
- i++;
- }
-
-
- } catch (SQLException e) {
- logger.error("Sqlexception: ", e);
- } finally {
- try {
- if (connection != null)
- connection.close();
- } catch (SQLException e) {
- logger.error("Sqlexception in closing the connection: ", e);
-
- }
- }
-
- return pigjobarraylist;
-
- }
-
- public void createDirPigJob(final String dir, final String namenodeuri) throws IOException,
- URISyntaxException {
-
- try {
- UserGroupInformation ugi = UserGroupInformation
- .createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
- Configuration conf = new Configuration();
- conf.set("fs.hdfs.impl",
- org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()
- );
- conf.set("fs.file.impl",
- org.apache.hadoop.fs.LocalFileSystem.class.getName()
- );
- conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
-
- FileSystem fs = FileSystem.get(conf);
- Path src = new Path(dir);
- fs.mkdirs(src);
- return null;
- }
- });
- } catch (Exception e) {
- logger.error("Webhdfs exception: ", e);
- }
- }
-
- /**/
- public void createDirPigJobSecured(final String dir, final String namenodeuri) throws IOException,
- URISyntaxException {
-
- try {
- final Configuration conf = new Configuration();
-
- conf.set("fs.hdfs.impl",
- org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()
- );
- conf.set("fs.file.impl",
- org.apache.hadoop.fs.LocalFileSystem.class.getName()
- );
- conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
- conf.set("hadoop.security.authentication", "Kerberos");
-
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
-
- FileSystem fs = FileSystem.get(conf);
- Path src = new Path(dir);
- fs.mkdirs(src);
- return null;
- }
- });
- } catch (Exception e) {
- logger.error("Webhdfs exception: ", e);
- }
- }
-
- /**/
- public void copyFileBetweenHdfs(final String source, final String dest, final String nameNodeuriAmbari, final String nameNodeuriHue)
- throws IOException {
-
- try {
- UserGroupInformation ugi = UserGroupInformation
- .createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
- Configuration confAmbari = new Configuration();
- confAmbari.set("fs.defaultFS", nameNodeuriAmbari);
- confAmbari.set("hadoop.job.ugi", "hdfs");
- FileSystem fileSystemAmbari = FileSystem.get(confAmbari);
-
- Configuration confHue = new Configuration();
- confHue.set("fs.defaultFS", nameNodeuriAmbari);
- confHue.set("hadoop.job.ugi", "hdfs");
- FileSystem fileSystemHue = FileSystem.get(confHue);
-
- String filename = source.substring(
- source.lastIndexOf('/') + 1, source.length());
- String dest1;
- if (dest.charAt(dest.length() - 1) != '/') {
- dest1 = dest + "/" + filename;
- } else {
- dest1 = dest + filename;
- }
-
- Path path1 = new Path(source);
- FSDataInputStream in1 = fileSystemHue.open(path1);
-
- Path path = new Path(dest1);
- if (fileSystemAmbari.exists(path)) {
-
- }
-
- FSDataOutputStream out = fileSystemAmbari.create(path);
-
- byte[] b = new byte[1024];
- int numBytes = 0;
- while ((numBytes = in1.read(b)) > 0) {
- out.write(b, 0, numBytes);
- }
- in1.close();
- out.close();
- fileSystemAmbari.close();
- return null;
- }
- });
- } catch (Exception e) {
- logger.error("Webhdfs exception: ", e);
- }
-
- }
-
- /**/
- public void copyFileBetweenHdfsSecured(final String source, final String dest, final String nameNodeuriAmbari, final String nameNodeuriHue)
- throws IOException {
-
- try {
-
- final Configuration confAmbari = new Configuration();
- confAmbari.set("fs.defaultFS", nameNodeuriAmbari);
- confAmbari.set("hadoop.job.ugi", "hdfs");
-
- final Configuration confHue = new Configuration();
- confHue.set("fs.defaultFS", nameNodeuriAmbari);
- confHue.set("hadoop.job.ugi", "hdfs");
-
- confAmbari.set("hadoop.security.authentication", "Kerberos");
- confHue.set("hadoop.security.authentication", "Kerberos");
-
- UserGroupInformation ugi = UserGroupInformation
- .createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
- FileSystem fileSystemAmbari = FileSystem.get(confAmbari);
-
- FileSystem fileSystemHue = FileSystem.get(confHue);
-
- String filename = source.substring(
- source.lastIndexOf('/') + 1, source.length());
- String dest1;
- if (dest.charAt(dest.length() - 1) != '/') {
- dest1 = dest + "/" + filename;
- } else {
- dest1 = dest + filename;
- }
-
- Path path1 = new Path(source);
- FSDataInputStream in1 = fileSystemHue.open(path1);
-
- Path path = new Path(dest1);
- if (fileSystemAmbari.exists(path)) {
-
- }
- FSDataOutputStream out = fileSystemAmbari.create(path);
- byte[] b = new byte[1024];
- int numBytes = 0;
- while ((numBytes = in1.read(b)) > 0) {
- out.write(b, 0, numBytes);
- }
- in1.close();
- out.close();
- fileSystemAmbari.close();
- return null;
- }
- });
- } catch (Exception e) {
- logger.error("Webhdfs exception: ", e);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
deleted file mode 100644
index 5d99b49..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobMigrationUtility.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.ambari.view.huetoambarimigration.migration.pig.pigjob;
-
-import java.beans.PropertyVetoException;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.text.ParseException;
-import java.util.ArrayList;
-
-import org.apache.ambari.view.ViewContext;
-
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.resources.PersonalCRUDResourceManager;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.MigrationResourceManager;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.PigModel;
-import org.apache.log4j.Logger;
-
-import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceAmbariDatabase;
-import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceHueDatabase;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.pig.jobqueryset.*;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.jobqueryset.*;
-
-public class PigJobMigrationUtility {
-
- protected MigrationResourceManager resourceManager = null;
-
- public synchronized PersonalCRUDResourceManager<MigrationResponse> getResourceManager(ViewContext view) {
- if (resourceManager == null) {
- resourceManager = new MigrationResourceManager(view);
- }
- return resourceManager;
- }
-
- public void pigJobMigration(String username, String instance, String startDate, String endDate, ViewContext view, MigrationResponse migrationresult, String jobid) throws IOException, ItemNotFound {
-
- long startTime = System.currentTimeMillis();
-
- final Logger logger = Logger.getLogger(PigJobMigrationUtility.class);
- Connection connectionHuedb = null;
- Connection connectionAmbaridb = null;
-
- logger.info("------------------------------");
- logger.info("pig Jobs Migration started");
- logger.info("------------------------------");
- logger.info("start date: " + startDate);
- logger.info("enddate date: " + endDate);
- logger.info("instance is: " + username);
- logger.info("hue username is : " + instance);
-
- PigJobMigrationImplementation pigjobimpl = new PigJobMigrationImplementation();// creating the implementation object
-
- QuerySet huedatabase=null;
-
- if(view.getProperties().get("huedrivername").contains("mysql"))
- {
- huedatabase=new MysqlQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("postgresql"))
- {
- huedatabase=new PostgressQuerySet();
- }
- else if(view.getProperties().get("huedrivername").contains("sqlite"))
- {
- huedatabase=new SqliteQuerySet();
- }
- else if (view.getProperties().get("huedrivername").contains("oracle"))
- {
- huedatabase=new OracleQuerySet();
- }
-
- QuerySetAmbariDB ambaridatabase=null;
-
- if(view.getProperties().get("ambaridrivername").contains("mysql"))
- {
- ambaridatabase=new MysqlQuerySetAmbariDB();
- }
- else if(view.getProperties().get("ambaridrivername").contains("postgresql"))
- {
- ambaridatabase=new PostgressQuerySetAmbariDB();
- }
- else if (view.getProperties().get("ambaridrivername").contains("oracle"))
- {
- ambaridatabase= new OracleQuerySetAmbariDB();
- }
- int maxCountforPigScript = 0,i=0;
-
- String time = null, timeIndorder = null;
- Long epochtime = null;
- String pigJobDirName;
- ArrayList<PigModel> pigJobDbPojo = new ArrayList<PigModel>();
-
- try {
-
- connectionHuedb = DataSourceHueDatabase.getInstance(view.getProperties().get("huedrivername"), view.getProperties().get("huejdbcurl"), view.getProperties().get("huedbusername"), view.getProperties().get("huedbpassword")).getConnection();//connecting to hue database
-
- pigJobDbPojo = pigjobimpl.fetchFromHueDB(username, startDate, endDate, connectionHuedb,huedatabase);// fetching the PigJobs details from hue
-
- for(int j=0;j<pigJobDbPojo.size();j++)
- {
- logger.info("the query fetched from hue="+pigJobDbPojo.get(i).getScript());
-
- }
-
- /*No pig Job details has been fetched accordring to search criteria*/
- if (pigJobDbPojo.size() == 0) {
-
- migrationresult.setIsNoQuerySelected("yes");
- migrationresult.setProgressPercentage(0);
- migrationresult.setNumberOfQueryTransfered(0);
- migrationresult.setTotalNoQuery(pigJobDbPojo.size());
- getResourceManager(view).update(migrationresult, jobid);
- logger.info("no pig Job has been selected from hue according to your criteria of searching");
-
- } else {
-
- connectionAmbaridb = DataSourceAmbariDatabase.getInstance(view.getProperties().get("ambaridrivername"), view.getProperties().get("ambarijdbcurl"), view.getProperties().get("ambaridbusername"), view.getProperties().get("ambaridbpassword")).getConnection();
- connectionAmbaridb.setAutoCommit(false);
-
- for (i = 0; i < pigJobDbPojo.size(); i++) {
-
- float calc = ((float) (i + 1)) / pigJobDbPojo.size() * 100;
- int progressPercentage = Math.round(calc);
- migrationresult.setIsNoQuerySelected("no");
- migrationresult.setProgressPercentage(progressPercentage);
- migrationresult.setNumberOfQueryTransfered(i+1);
- migrationresult.setTotalNoQuery(pigJobDbPojo.size());
- getResourceManager(view).update(migrationresult, jobid);
-
-
-
-
- logger.info("Loop No." + (i + 1));
- logger.info("________________");
- logger.info("the title of script " + pigJobDbPojo.get(i).getTitle());
-
- int fetchPigTablenameInstance = pigjobimpl.fetchInstanceTablename(connectionAmbaridb, instance,ambaridatabase);
-
- maxCountforPigScript = (pigjobimpl.fetchMaxIdforPigJob(connectionAmbaridb, fetchPigTablenameInstance,ambaridatabase) + 1);
-
- time = pigjobimpl.getTime();
- timeIndorder = pigjobimpl.getTimeInorder();
- epochtime = pigjobimpl.getEpochTime();
-
- pigJobDirName = "/user/admin/pig/jobs/" + pigJobDbPojo.get(i).getTitle() + "_" + time + "/";
-
- pigjobimpl.insertRowPigJob(pigJobDirName, maxCountforPigScript, time, timeIndorder, epochtime, pigJobDbPojo.get(i).getTitle(), connectionAmbaridb, fetchPigTablenameInstance, pigJobDbPojo.get(i).getStatus(), instance, i,ambaridatabase);
-
- if (view.getProperties().get("KerberoseEnabled").equals("y")) {
-
- pigjobimpl.createDirPigJobSecured(pigJobDirName, view.getProperties().get("namenode_URI_Ambari"));
- pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/script.pig", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/stderr", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfsSecured(pigJobDbPojo.get(i).getDir() + "/stdout", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
-
- } else {
-
- pigjobimpl.createDirPigJob(pigJobDirName, view.getProperties().get("namenode_URI_Ambari"));
- pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/script.pig", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/stderr", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
- pigjobimpl.copyFileBetweenHdfs(pigJobDbPojo.get(i).getDir() + "/stdout", pigJobDirName, view.getProperties().get("namenode_URI_Ambari"), view.getProperties().get("namenode_URI_Hue"));
-
- }
-
- logger.info(pigJobDbPojo.get(i).getTitle() + "has been migrated to Ambari");
-
- }
- connectionAmbaridb.commit();
- }
-
- } catch (SQLException e) {
- logger.error("sql exception in ambari database:", e);
- try {
- connectionAmbaridb.rollback();
- logger.info("roll back done");
- } catch (SQLException e1) {
- logger.error("roll back exception:",e1);
- }
- } catch (ClassNotFoundException e2) {
- logger.error("class not found exception:",e2);
- } catch (ParseException e) {
- logger.error("ParseException: " ,e);
- } catch (URISyntaxException e) {
- logger.error("URISyntaxException" ,e);
- } catch (PropertyVetoException e) {
- logger.error("PropertyVetoException" ,e);
- } finally {
- if (null != connectionAmbaridb)
- try {
- connectionAmbaridb.close();
- } catch (SQLException e) {
- logger.error("connection closing exception ", e);
- }
- }
-
- logger.info("------------------------------");
- logger.info("pig Job Migration End");
- logger.info("------------------------------");
-
- //session.setAttribute(ProgressBarStatus.TASK_PROGRESS_VARIABLE, 0);
-
-// CheckProgresStatus.setProgressPercentage(0);
-// CheckProgresStatus.setNoOfQueryCompleted(0);
-// CheckProgresStatus.setTotalNoOfQuery(0);
-// CheckProgresStatus.setNoOfQueryLeft(0);
-
- long stopTime = System.currentTimeMillis();
- long elapsedTime = stopTime - startTime;
-
- migrationresult.setJobtype("hivehistoryquerymigration");
- migrationresult.setTotalTimeTaken(String.valueOf(elapsedTime));
- getResourceManager(view).update(migrationresult, jobid);
-
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobStartJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobStartJob.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobStartJob.java
deleted file mode 100644
index 15f033f..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigjob/PigJobStartJob.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.view.huetoambarimigration.migration.pig.pigjob;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.json.simple.JSONObject;
-
-import java.io.IOException;
-
-
-public class PigJobStartJob extends Thread{
-
- String username;
- String instance;
- String startdate;
- String enddate;
- String jobid;
- ViewContext view;
-
- public PigJobStartJob(String username, String instance, String startdate, String enddate, String jobid, ViewContext view) {
- this.username = username;
- this.instance=instance;
- this.startdate=startdate;
- this.enddate=enddate;
- this.jobid=jobid;
- this.view=view;
- }
-
- @Override
- public void run() {
-
- MigrationResponse migrationresult=new MigrationResponse();
-
- migrationresult.setId(jobid);
- migrationresult.setIntanceName(instance);
- migrationresult.setUserNameofhue(username);
- migrationresult.setProgressPercentage(0);
-
- PigJobMigrationUtility pigjobmigration=new PigJobMigrationUtility();
- try {
- pigjobmigration.pigJobMigration(username,instance,startdate,enddate,view,migrationresult,jobid);
- }
- catch (IOException e) {
- e.printStackTrace();
- } catch (ItemNotFound itemNotFound) {
- itemNotFound.printStackTrace();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
deleted file mode 100644
index c5f073c..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigSavedScriptStartJob.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.view.huetoambarimigration.migration.pig.pigscript;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.json.simple.JSONObject;
-
-import java.io.IOException;
-
-
-public class PigSavedScriptStartJob extends Thread{
-
- String username;
- String instance;
- String startdate;
- String enddate;
- String jobid;
- ViewContext view;
-
- public PigSavedScriptStartJob(String username, String instance, String startdate, String enddate, String jobid, ViewContext view) {
- this.username = username;
- this.instance=instance;
- this.startdate=startdate;
- this.enddate=enddate;
- this.jobid=jobid;
- this.view=view;
- }
-
-
-
- @Override
- public void run() {
-
- MigrationResponse migrationresult=new MigrationResponse();
-
- migrationresult.setId(jobid);
- migrationresult.setIntanceName(instance);
- migrationresult.setUserNameofhue(username);
- migrationresult.setProgressPercentage(0);
-
- PigScriptMigrationUtility pigsavedscript =new PigScriptMigrationUtility();
- try {
- pigsavedscript.pigScriptMigration(username,instance,startdate,enddate,view,migrationresult,jobid);
- }
- catch (IOException e) {
- e.printStackTrace();
- } catch (ItemNotFound itemNotFound) {
- itemNotFound.printStackTrace();
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
deleted file mode 100644
index c8aa1c0..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationImplementation.java
+++ /dev/null
@@ -1,504 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.ambari.view.huetoambarimigration.migration.pig.pigscript;
-
-import org.apache.ambari.view.huetoambarimigration.migration.pig.pigjob.PigJobMigrationImplementation;
-import org.apache.ambari.view.huetoambarimigration.migration.configuration.ConfigurationCheckImplementation;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.PigModel;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.savedscriptqueryset.QuerySetAmbariDB;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.pig.savedscriptqueryset.QuerySet;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Logger;
-import org.jdom.Attribute;
-import org.jdom.Document;
-import org.jdom.Element;
-import org.jdom.JDOMException;
-import org.jdom.input.SAXBuilder;
-import org.jdom.output.Format;
-import org.jdom.output.XMLOutputter;
-
-import java.io.*;
-import java.security.PrivilegedExceptionAction;
-import java.sql.*;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.GregorianCalendar;
-
-
-public class PigScriptMigrationImplementation {
-
- static final Logger logger = Logger.getLogger(PigJobMigrationImplementation.class);
-
- private static String readAll(Reader rd) throws IOException {
- StringBuilder sb = new StringBuilder();
- int cp;
- while ((cp = rd.read()) != -1) {
- sb.append((char) cp);
- }
- return sb.toString();
- }
-
- public void wrtitetoalternatesqlfile(String dirname, String content, String instance, int i) throws IOException {
-
- Date dNow = new Date();
- SimpleDateFormat ft = new SimpleDateFormat("YYYY-MM-dd hh:mm:ss");
- String currentDate = ft.format(dNow);
-
- XMLOutputter xmlOutput = new XMLOutputter();
-
- xmlOutput.setFormat(Format.getPrettyFormat());
-
- File xmlfile = new File(ConfigurationCheckImplementation.getHomeDir() + "RevertChangesService.xml");
-
- if (xmlfile.exists()) {
- String iteration = Integer.toString(i + 1);
- SAXBuilder builder = new SAXBuilder();
- Document doc;
- try {
- doc = (Document) builder.build(xmlfile);
-
- Element rootNode = doc.getRootElement();
-
- Element record = new Element("RevertRecord");
- record.setAttribute(new Attribute("id", iteration));
- record.addContent(new Element("datetime").setText(currentDate.toString()));
- record.addContent(new Element("dirname").setText(dirname));
- record.addContent(new Element("instance").setText(instance));
- record.addContent(new Element("query").setText(content));
-
- rootNode.addContent(record);
- xmlOutput.output(doc, new FileWriter(ConfigurationCheckImplementation.getHomeDir() + "RevertChangesService.xml"));
-
- } catch (JDOMException e) {
- logger.error("JDOMException: ", e);
- }
-
-
- } else {
- // create
- try {
- String iteration = Integer.toString(i + 1);
- Element revertrecord = new Element("RevertChangePage");
- Document doc = new Document(revertrecord);
- doc.setRootElement(revertrecord);
-
- Element record = new Element("RevertRecord");
- record.setAttribute(new Attribute("id", iteration));
- record.addContent(new Element("datetime").setText(currentDate.toString()));
- record.addContent(new Element("dirname").setText(dirname));
- record.addContent(new Element("instance").setText(instance));
- record.addContent(new Element("query").setText(content));
-
- doc.getRootElement().addContent(record);
-
- xmlOutput.output(doc, new FileWriter(ConfigurationCheckImplementation.getHomeDir() + "RevertChangesService.xml"));
-
- } catch (IOException io) {
- logger.error("IOException: ", io);
-
- }
-
- }
-
-
- }
-
- public int fetchInstanceTablenamePigScript(Connection c, String instance, QuerySetAmbariDB ambaridatabase) throws SQLException {
-
- String ds_id = new String();
- int id = 0;
- Statement stmt = null;
- PreparedStatement prSt = null;
-
-
- ResultSet rs = null;
-
-
- prSt = ambaridatabase.getTableIdFromInstanceName(c, instance);
-
- logger.info("sql statement to fetch is from ambari instance:= = " + prSt);
-
- rs = prSt.executeQuery();
-
- while (rs.next()) {
- id = rs.getInt("id");
- }
- return id;
-
- }
-
- public int fetchmaxIdforPigSavedScript(Connection c, int id, QuerySetAmbariDB ambaridatabase) throws SQLException {
-
-
- String ds_id = null;
- ResultSet rs = null;
- PreparedStatement prSt = null;
-
- prSt = ambaridatabase.getMaxDsIdFromTableId(c, id);
-
- rs = prSt.executeQuery();
-
- while (rs.next()) {
- ds_id = rs.getString("max");
- }
-
- int num;
- if (ds_id == null) {
- num = 1;
- } else {
- num = Integer.parseInt(ds_id);
- }
- return num;
- }
-
- public void insertRowForPigScript(String dirname, int maxcountforpigjob, int maxcount, String time, String time2, long epochtime, String title, Connection c, int id, String instance, int i, QuerySetAmbariDB ambaridatabase) throws SQLException, IOException {
-
- String maxcount1 = Integer.toString(maxcount);
- String epochtime1 = Long.toString(epochtime);
- String revSql = null;
-
- PreparedStatement prSt = null;
-
- prSt = ambaridatabase.insertToPigScript(c, id, maxcount1, dirname, title);
-
- prSt.executeUpdate();
-
- revSql = ambaridatabase.revertSql(id, maxcount1);
-
- wrtitetoalternatesqlfile(dirname, revSql, instance, i);
-
- }
-
-
- public long getEpochTime() throws ParseException {
- int day, month, year;
- int second, minute, hour;
- int milisecond;
- GregorianCalendar date = new GregorianCalendar();
-
- day = date.get(Calendar.DAY_OF_MONTH);
- month = date.get(Calendar.MONTH);
- year = date.get(Calendar.YEAR);
-
- second = date.get(Calendar.SECOND);
- minute = date.get(Calendar.MINUTE);
- hour = date.get(Calendar.HOUR);
- milisecond = date.get(Calendar.MILLISECOND);
-
- String s1 = year + "-" + (month + 1) + "-" + day + "_" + hour + "-" + minute + "-" + second + "-" + milisecond;
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS");
- Date date1 = df.parse(s1);
- long epoch = date1.getTime();
-
- return epoch;
-
- }
-
-
- public String getTime() throws ParseException {
- int day, month, year;
- int second, minute, hour;
- int milisecond;
- GregorianCalendar date = new GregorianCalendar();
-
- day = date.get(Calendar.DAY_OF_MONTH);
- month = date.get(Calendar.MONTH);
- year = date.get(Calendar.YEAR);
-
- second = date.get(Calendar.SECOND);
- minute = date.get(Calendar.MINUTE);
- hour = date.get(Calendar.HOUR);
- milisecond = date.get(Calendar.MILLISECOND);
-
- String s = year + "-" + (month + 1) + "-" + day + "_" + hour + "-" + minute;
- String s1 = year + "-" + (month + 1) + "-" + day + "_" + hour + "-" + minute + "-" + second + "-" + milisecond;
- SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-SSS");
- Date date1 = df.parse(s1);
- long epoch = date1.getTime();
-
- return s;
-
- }
-
-
- public String getTimeInorder() throws ParseException {
- SimpleDateFormat sdfDate = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.msssss +00:00:00");//dd/MM/yyyy
- Date now = new Date();
- String strDate = sdfDate.format(now);
- return strDate;
- }
-
-
- public ArrayList<PigModel> fetchFromHueDatabase(String username, String startdate, String endtime, Connection connection, QuerySet huedatabase) throws ClassNotFoundException, IOException {
- int id = 0;
- int i = 0;
- ResultSet rs1 = null;
- String[] query = new String[100];
- ArrayList<PigModel> pigArrayList = new ArrayList<PigModel>();
- try {
- Statement statement = connection.createStatement();
- connection.setAutoCommit(false);
- PreparedStatement prSt = null;
- ResultSet rs;
- if (username.equals("all")) {
- } else {
-
- prSt = huedatabase.getUseridfromUserName(connection, username);
-
- rs = prSt.executeQuery();
-
- while (rs.next()) {
- id = rs.getInt("id");
- }
- }
-
- if (startdate.equals("") && endtime.equals("")) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesNoStartDateNoEndDateAllUser(connection);
- } else {
- prSt = huedatabase.getQueriesNoStartDateNoEndDate(connection, id);
-
- }
-
- } else if ((startdate.equals("")) && !(endtime.equals(""))) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesNoStartDateYesEndDateAllUser(connection, endtime);
- } else {
- prSt = huedatabase.getQueriesNoStartDateYesEndDate(connection, id, endtime);
-
- }
- } else if (!(startdate.equals("")) && (endtime.equals(""))) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesYesStartDateNoEndDateAllUser(connection, startdate);
- } else {
- prSt = huedatabase.getQueriesYesStartDateNoEndDate(connection, id, startdate);
-
- }
-
- } else if (!(startdate.equals("")) && !(endtime.equals(""))) {
- if (username.equals("all")) {
- prSt = huedatabase.getQueriesYesStartDateYesEndDateAllUser(connection, startdate, endtime);
- } else {
- prSt = huedatabase.getQueriesYesStartDateYesEndDate(connection, id, startdate, endtime);
- }
-
-
- }
-
- rs1 = prSt.executeQuery();
-
-
- // rs1 = statement.executeQuery("select pig_script,title,date_created,saved,arguments from pig_pigscript where saved=1 AND user_id ="+id+" AND date_created BETWEEN '"+ startdate +"' AND '" +endtime +"';");
- while (rs1.next()) {
- PigModel pojopig = new PigModel();
- String script = rs1.getString("pig_script");
- String title = rs1.getString("title");
- Date created_data = rs1.getDate("date_created");
- pojopig.setDt(created_data);
- pojopig.setScript(script);
- pojopig.setTitle(title);
-
- pigArrayList.add(pojopig);
- i++;
- }
-
-
- } catch (SQLException e) {
- logger.error("SQLException", e);
- } finally {
- try {
- if (connection != null)
- connection.close();
- } catch (SQLException e) {
- logger.error("SQLException", e);
- }
- }
-
- return pigArrayList;
-
- }
-
- public void writetPigScripttoLocalFile(String script, String title, Date createddate, String homedir, String filename2) {
- try {
- logger.info(homedir + filename2);
- File file = new File(homedir + filename2);
-
- if (!file.exists()) {
- file.createNewFile();
- }
-
- FileWriter fw = new FileWriter(file.getAbsoluteFile());
- BufferedWriter bw = new BufferedWriter(fw);
- bw.write(script);
- bw.close();
-
-
- } catch (IOException e) {
-
- logger.error("IOException", e);
- }
-
- }
-
- public void deletePigScriptLocalFile(String homedir, String filename2) {
- try {
-
- File file = new File(homedir + filename2);
-
- if (file.delete()) {
- logger.info("Temproray file deleted");
- } else {
- logger.info("Temproray file delete failed");
- }
-
- } catch (Exception e) {
-
- logger.error("File Exception: ", e);
-
- }
-
- }
-
- public void putFileinHdfs(final String source, final String dest, final String namenodeuri)
- throws IOException {
-
- try {
- UserGroupInformation ugi = UserGroupInformation
- .createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
- Configuration conf = new Configuration();
- conf.set("fs.hdfs.impl",
- org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()
- );
- conf.set("fs.file.impl",
- org.apache.hadoop.fs.LocalFileSystem.class.getName()
- );
- conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
- FileSystem fileSystem = FileSystem.get(conf);
-
- String filename = source.substring(
- source.lastIndexOf('/') + 1, source.length());
- String dest1;
- if (dest.charAt(dest.length() - 1) != '/') {
- dest1 = dest + "/" + filename;
- } else {
- dest1 = dest + filename;
- }
-
- Path path = new Path(dest1);
- if (fileSystem.exists(path)) {
-
- }
- FSDataOutputStream out = fileSystem.create(path);
-
- InputStream in = new BufferedInputStream(
- new FileInputStream(new File(source)));
-
- byte[] b = new byte[1024];
- int numBytes = 0;
- while ((numBytes = in.read(b)) > 0) {
- out.write(b, 0, numBytes);
- }
- in.close();
- out.close();
- fileSystem.close();
- return null;
- }
- });
- } catch (Exception e) {
- logger.error("Webhdfs Exception: ", e);
- }
-
- }
-
- public void putFileinHdfsSecured(final String source, final String dest, final String namenodeuri)
- throws IOException {
-
- try {
- final Configuration conf = new Configuration();
-
- conf.set("fs.hdfs.impl",
- org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()
- );
- conf.set("fs.file.impl",
- org.apache.hadoop.fs.LocalFileSystem.class.getName()
- );
- conf.set("fs.defaultFS", namenodeuri);
- conf.set("hadoop.job.ugi", "hdfs");
- conf.set("hadoop.security.authentication", "Kerberos");
-
- UserGroupInformation.setConfiguration(conf);
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser("hdfs");
-
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- public Void run() throws Exception {
-
- FileSystem fileSystem = FileSystem.get(conf);
-
- String filename = source.substring(
- source.lastIndexOf('/') + 1, source.length());
- String dest1;
- if (dest.charAt(dest.length() - 1) != '/') {
- dest1 = dest + "/" + filename;
- } else {
- dest1 = dest + filename;
- }
-
- Path path = new Path(dest1);
- if (fileSystem.exists(path)) {
-
- }
- // Path pathsource = new Path(source);
- FSDataOutputStream out = fileSystem.create(path);
-
- InputStream in = new BufferedInputStream(
- new FileInputStream(new File(source)));
-
- byte[] b = new byte[1024];
- int numBytes = 0;
- while ((numBytes = in.read(b)) > 0) {
- out.write(b, 0, numBytes);
- }
- in.close();
- out.close();
- fileSystem.close();
- return null;
- }
- });
- } catch (Exception e) {
- logger.error("Webhdfs Exception: ", e);
-
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
deleted file mode 100644
index 44e27c1..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/pig/pigscript/PigScriptMigrationUtility.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.ambari.view.huetoambarimigration.migration.pig.pigscript;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceAmbariDatabase;
-import org.apache.ambari.view.huetoambarimigration.datasource.DataSourceHueDatabase;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.PigModel;
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.ambariqueryset.pig.savedscriptqueryset.*;
-import org.apache.ambari.view.huetoambarimigration.datasource.queryset.huequeryset.pig.savedscriptqueryset.*;
-import org.apache.ambari.view.huetoambarimigration.resources.PersonalCRUDResourceManager;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.MigrationResourceManager;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.apache.ambari.view.huetoambarimigration.migration.configuration.ConfigurationCheckImplementation;
-import org.apache.log4j.Logger;
-
-import java.beans.PropertyVetoException;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.text.ParseException;
-import java.util.ArrayList;
-
-public class PigScriptMigrationUtility {
-
- protected MigrationResourceManager resourceManager = null;
-
- public synchronized PersonalCRUDResourceManager<MigrationResponse> getResourceManager(ViewContext view) {
- if (resourceManager == null) {
- resourceManager = new MigrationResourceManager(view);
- }
- return resourceManager;
- }
-
-
- public void pigScriptMigration(String username, String instance, String startDate, String endDate, ViewContext view, MigrationResponse migrationresult, String jobid) throws IOException, ItemNotFound {
-
- long startTime = System.currentTimeMillis();
-
- final Logger logger = Logger.getLogger(PigScriptMigrationUtility.class);
- Connection connectionHuedb = null;
- Connection connectionAmbaridb = null;
-
- logger.info("-------------------------------------");
- logger.info("pig saved script Migration started");
- logger.info("-------------------------------------");
-
-
- int i = 0;
-
- logger.info("start date: " + startDate);
- logger.info("enddate date: " + endDate);
- logger.info("instance is: " + username);
- logger.info("hue username is : " + instance);
-
- //Reading the configuration file
- PigScriptMigrationImplementation pigsavedscriptmigration = new PigScriptMigrationImplementation();
-
- QuerySet huedatabase = null;
-
- if (view.getProperties().get("huedrivername").contains("mysql")) {
- huedatabase = new MysqlQuerySet();
- } else if (view.getProperties().get("huedrivername").contains("postgresql")) {
- huedatabase = new PostgressQuerySet();
- } else if (view.getProperties().get("huedrivername").contains("sqlite")) {
-
- huedatabase = new SqliteQuerySet();
- } else if (view.getProperties().get("huedrivername").contains("oracle")) {
- huedatabase = new OracleQuerySet();
- }
-
- QuerySetAmbariDB ambaridatabase = null;
-
-
- if (view.getProperties().get("ambaridrivername").contains("mysql")) {
- ambaridatabase = new MysqlQuerySetAmbariDB();
- } else if (view.getProperties().get("ambaridrivername").contains("postgresql")) {
- ambaridatabase = new PostgressQuerySetAmbariDB();
- } else if (view.getProperties().get("ambaridrivername").contains("oracle")) {
- ambaridatabase = new OracleQuerySetAmbariDB();
- }
-
- int maxcountforsavequery = 0, maxcountforpigsavedscript;
- String time = null, timetobeInorder = null;
- Long epochTime = null;
- String dirNameForPigScript, completeDirandFilePath, pigscriptFilename = "";
- int pigInstanceTableName;
-
- ArrayList<PigModel> dbpojoPigSavedscript = new ArrayList<PigModel>();
-
- try {
- connectionHuedb = DataSourceHueDatabase.getInstance(view.getProperties().get("huedrivername"), view.getProperties().get("huejdbcurl"), view.getProperties().get("huedbusername"), view.getProperties().get("huedbpassword")).getConnection();//connection to Hue DB
- dbpojoPigSavedscript = pigsavedscriptmigration.fetchFromHueDatabase(username, startDate, endDate, connectionHuedb, huedatabase);// Fetching pig script details from Hue DB
-
- for (int j = 0; j < dbpojoPigSavedscript.size(); j++) {
- logger.info("the query fetched from hue=" + dbpojoPigSavedscript.get(j).getScript());
-
- }
-
-
- /* If No pig Script has been fetched from Hue db according to our search criteria*/
- if (dbpojoPigSavedscript.size() == 0) {
-
- migrationresult.setIsNoQuerySelected("yes");
- migrationresult.setProgressPercentage(0);
- migrationresult.setNumberOfQueryTransfered(0);
- migrationresult.setTotalNoQuery(dbpojoPigSavedscript.size());
- getResourceManager(view).update(migrationresult, jobid);
-
- logger.info("no pig script has been selected from hue according to your criteria of searching");
-
-
- } else {
-
- connectionAmbaridb = DataSourceAmbariDatabase.getInstance(view.getProperties().get("ambaridrivername"), view.getProperties().get("ambarijdbcurl"), view.getProperties().get("ambaridbusername"), view.getProperties().get("ambaridbpassword")).getConnection();// connecting to ambari db
- connectionAmbaridb.setAutoCommit(false);
- logger.info("loop will continue for " + dbpojoPigSavedscript.size() + "times");
-
- //for each pig script found in Hue Database
-
- for (i = 0; i < dbpojoPigSavedscript.size(); i++) {
-
-
- float calc = ((float) (i + 1)) / dbpojoPigSavedscript.size() * 100;
- int progressPercentage = Math.round(calc);
- migrationresult.setIsNoQuerySelected("no");
- migrationresult.setProgressPercentage(progressPercentage);
- migrationresult.setNumberOfQueryTransfered(i + 1);
- migrationresult.setTotalNoQuery(dbpojoPigSavedscript.size());
- getResourceManager(view).update(migrationresult, jobid);
-
- logger.info("Loop No." + (i + 1));
- logger.info("________________");
- logger.info("the title of script: " + dbpojoPigSavedscript.get(i).getTitle());
-
- pigInstanceTableName = pigsavedscriptmigration.fetchInstanceTablenamePigScript(connectionAmbaridb, instance, ambaridatabase);// finding the table name in ambari from the given instance
-
- maxcountforpigsavedscript = (pigsavedscriptmigration.fetchmaxIdforPigSavedScript(connectionAmbaridb, pigInstanceTableName, ambaridatabase) + 1);// maximum count of the primary key of pig Script table
-
- time = pigsavedscriptmigration.getTime();
-
- timetobeInorder = pigsavedscriptmigration.getTimeInorder();
-
- epochTime = pigsavedscriptmigration.getEpochTime();
-
- dirNameForPigScript = "/user/admin/pig/scripts/";
-
- pigscriptFilename = dbpojoPigSavedscript.get(i).getTitle() + "-" + time + ".pig";
-
- completeDirandFilePath = dirNameForPigScript + pigscriptFilename;
-
- pigsavedscriptmigration.writetPigScripttoLocalFile(dbpojoPigSavedscript.get(i).getScript(), dbpojoPigSavedscript.get(i).getTitle(), dbpojoPigSavedscript.get(i).getDt(), ConfigurationCheckImplementation.getHomeDir(), pigscriptFilename);
-
- pigsavedscriptmigration.insertRowForPigScript(completeDirandFilePath, maxcountforsavequery, maxcountforpigsavedscript, time, timetobeInorder, epochTime, dbpojoPigSavedscript.get(i).getTitle(), connectionAmbaridb, pigInstanceTableName, instance, i, ambaridatabase);
-
- if (view.getProperties().get("KerberoseEnabled").equals("y")) {
- pigsavedscriptmigration.putFileinHdfsSecured(ConfigurationCheckImplementation.getHomeDir() + pigscriptFilename, dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"));
- } else {
- pigsavedscriptmigration.putFileinHdfs(ConfigurationCheckImplementation.getHomeDir() + pigscriptFilename, dirNameForPigScript, view.getProperties().get("namenode_URI_Ambari"));
- }
-
- logger.info(dbpojoPigSavedscript.get(i).getTitle() + "Migrated to Ambari");
-
- pigsavedscriptmigration.deletePigScriptLocalFile(ConfigurationCheckImplementation.getHomeDir(), pigscriptFilename);
-
- }
- connectionAmbaridb.commit();
-
- }
-
-
- } catch (SQLException e) {
- logger.error("Sql exception in ambari database", e);
- try {
- connectionAmbaridb.rollback();
- logger.info("rollback done");
- } catch (SQLException e1) {
- logger.error("Sql exception while doing roll back", e);
- }
- } catch (ClassNotFoundException e2) {
- logger.error("class not found exception", e2);
- } catch (ParseException e) {
- logger.error("ParseException: ", e);
- } catch (PropertyVetoException e) {
- logger.error("PropertyVetoException: ", e);
- } finally {
- if (null != connectionAmbaridb)
- try {
- connectionAmbaridb.close();
- } catch (SQLException e) {
- logger.error("connection close exception: ", e);
- }
- }
-
- long stopTime = System.currentTimeMillis();
- long elapsedTime = stopTime - startTime;
-
-
- migrationresult.setJobtype("hivehistoryquerymigration");
- migrationresult.setTotalTimeTaken(String.valueOf(elapsedTime));
- getResourceManager(view).update(migrationresult, jobid);
-
-
- logger.info("----------------------------------");
- logger.info("pig saved script Migration ends");
- logger.info("----------------------------------");
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c40ec582/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/revertchange/RevertChangeStartJob.java
----------------------------------------------------------------------
diff --git a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/revertchange/RevertChangeStartJob.java b/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/revertchange/RevertChangeStartJob.java
deleted file mode 100644
index 31cec2f..0000000
--- a/contrib/views/hueambarimigration/src/main/java/org/apache/ambari/view/huetoambarimigration/migration/revertchange/RevertChangeStartJob.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ambari.view.huetoambarimigration.migration.revertchange;
-
-import org.apache.ambari.view.ViewContext;
-import org.apache.ambari.view.huetoambarimigration.persistence.utils.ItemNotFound;
-import org.apache.ambari.view.huetoambarimigration.resources.scripts.models.MigrationResponse;
-import org.json.simple.JSONObject;
-
-import java.io.IOException;
-
-
-public class RevertChangeStartJob extends Thread{
-
-
- String instance;
- String revertdate;
- String jobid;
- ViewContext view;
-
- public RevertChangeStartJob(String instance, String revertdate, String jobid, ViewContext view) {
-
- this.instance=instance;
- this.revertdate=revertdate;
- this.jobid=jobid;
- this.view=view;
- }
-
-
-
- @Override
- public void run() {
-
- MigrationResponse migrationresult=new MigrationResponse();
-
- migrationresult.setId(jobid);
- migrationresult.setIntanceName(instance);
- migrationresult.setProgressPercentage(0);
-
- JSONObject response = new JSONObject();
-
-
- RevertChangeUtility revertchange = new RevertChangeUtility();
- try {
- revertchange.revertChangeUtility(instance,revertdate,jobid,view,migrationresult);
- }
- catch (IOException e) {
- e.printStackTrace();
- } catch (ItemNotFound itemNotFound) {
- itemNotFound.printStackTrace();
- }
-
- }
-
-}